diff --git a/wandb/run-20260319_223122-agxw5i56/files/code/train/sft_lladav_video_reversal.py b/wandb/run-20260319_223122-agxw5i56/files/code/train/sft_lladav_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..595bb1a900f57c194edae31b5826e22f2b5c9a05 --- /dev/null +++ b/wandb/run-20260319_223122-agxw5i56/files/code/train/sft_lladav_video_reversal.py @@ -0,0 +1,750 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator, InitProcessGroupKwargs +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +# LLaDA-V / LLaVA +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_pyav, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates +# from accelerate import DistributedDataParallelKwargs #添加 + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) # (L,) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + """ + 统一处理 Image 和 Video + 返回 (media_tensor_list, sizes, video_stats) + """ + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 0) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", False) + + # 使用官方提供的 decord 函数处理视频 + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + + # 处理成 tensor + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + # 提取 size + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + """通用媒体路径解析""" + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class SFTVariantDatasetLLADAV(Dataset): + + def __init__(self, samples: List[dict], method: str, lower: float, upper: float, block_size: int, mask_times: int, max_gen_length: int): + super().__init__() + self.samples = samples + self.method = method + self.lower = lower + self.upper = upper + self.block_size = block_size + self.mask_times = mask_times + self.max_gen_length = max_gen_length + + self.variants: List[dict] = [] + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt="\nOnly give the best option." + + rng = np.random.default_rng(seed=42) + for sid, s in enumerate(self.samples): + if self.method == "semi-ar": + step_map = s.get("step_map", None) + if step_map is None: + step_map = list(range(self.max_gen_length)) + order_list = collapse_k_unique(step_map, self.block_size) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(dict( + sample_id=sid, + prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt, + response=s["response"], + image_abs_path=s.get("image_abs_path", None), + image=s.get("image", None), + image_path=s.get("image_path", None), + video_abs_path=s.get("video_abs_path", None), + video=s.get("video", None), + video_path=s.get("video_path", None), + var_type="semi_ar", + var_param=int(st), + step_map=order_list, + mask_seed=None, + )) + elif self.method == "random_masking": + for _ in range(self.mask_times): + t = float(rng.uniform(self.lower, self.upper)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(dict( + sample_id=sid, + prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt, + response=s["response"], + image_abs_path=s.get("image_abs_path", None), + image=s.get("image", None), + image_path=s.get("image_path", None), + video_abs_path=s.get("video_abs_path", None), + video=s.get("video", None), + video_path=s.get("video_path", None), + var_type="random_masking", + var_param=t, + step_map=None, + mask_seed=mask_seed, + )) + else: + raise ValueError(f"Unknown SFT method: {self.method}") + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + v = self.variants[idx] + return { + "idx": idx, + "sample_id": v["sample_id"], + "prompt": v["prompt"], + "response": v["response"], + "image_abs_path": v.get("image_abs_path", None), + "image": v.get("image", None), + "image_path": v.get("image_path", None), + "video_abs_path": v.get("video_abs_path", None), + "video": v.get("video", None), + "video_path": v.get("video_path", None), + "var_type": v["var_type"], + "var_param": v["var_param"], + "step_map": v.get("step_map", None), + "mask_seed": v.get("mask_seed", None), + } + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "image": batch.get("image", [None])[first_pos], + "image_path": batch.get("image_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + "video": batch.get("video", [None])[first_pos], + "video_path": batch.get("video_path", [None])[first_pos], + } + + # 优先检测视频,其次检测图片 + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + + modalities = [media_type] + + # 若包含视频且需附加时间指引,将参数注入 prompt + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + # 剔除原有可能存在的图片 Token 以免重叠 + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + # Tokenize (内部会自动 prepend DEFAULT_IMAGE_TOKEN) + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) # (L, D) + new_labels = new_labels.squeeze(0) # (L,) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) # (M,) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [] + for i in range(bsz): + sid = int(batch["sample_id"][i]) + lengths.append(int(sample_cache[sid]["clean_embeds"].shape[0])) + Lmax = max(lengths) + D = next(iter(sample_cache.values()))["clean_embeds"].shape[1] + + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list = [] + labels_list = [] + p_mask_list = [] + attn_list = [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + step_map = batch["step_map"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + + if var_type == "semi_ar": + order = torch.as_tensor(step_map, device=device) + M = asst_idx.numel() + if order.numel() != M: + m = min(M, order.numel()) + asst_idx = asst_idx[:m] + order = order[:m] + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx] = order + step_val = int(var_param) + + p = torch.empty(L, device=device).uniform_(0.0, 1.0) + lower = 0.0 + 1e-6 + upper = 1.0 - 1e-6 + thresh = (upper - lower) * torch.rand(1, device=device) + lower + block_mask = (order_full == step_val) & (p < thresh) + + mask_pos = (order_full > step_val) | block_mask + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + pmask = block_mask + + elif var_type == "random_masking": + M = asst_idx.numel() + if M == 0: + noisy_embeds = clean_embeds + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + gen = torch.Generator(device="cpu") + gen.manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + rnd = rnd_cpu.to(device=device) + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + else: + continue + + if not pmask.any(): + continue + # if not pmask.any(): + # # print('*****no token masked*****') + # if asst_idx.numel() > 0: + # # 强制 Mask 最后一个 assistant token + # force_idx = asst_idx[-1] + # pmask[force_idx] = True + # noisy_embeds[force_idx] = mask_embed + # else: + # print("*********bad data*********") + # print(batch["sample_id"][i]) + # # 如果连 assistant token 都没有,那这条数据本身就是坏的,只能跳过 + # continue + + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + return noisy_batch, labels_batch, p_mask_batch, attn_batch + +# ... 余下 save_checkpoint 和 main() 训练循环逻辑无需修改,保持原样即可 ... +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + # TF32 + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + + # 🌟 实例化 DDP 处理器,允许存在未使用的参数 + # ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True) + # process_group_kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=3600)) + + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with="wandb", + project_dir=config.experiment.logging_dir, + split_batches=True, + # kwargs_handlers=[process_group_kwargs] # 🌟 把超时设置传进去 + # kwargs_handlers=[ddp_kwargs], # <-- 把这个参数传进去 + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + pretrained_model = config.model.pretrained_model + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + with open("./data/" + config.dataset.optimization_data + ".json", 'r') as f: + data_list = json.load(f) + + ds = SFTVariantDatasetLLADAV( + samples=data_list, + method=config.training.method, + lower=config.training.lower_p, + upper=config.training.upper_p, + block_size=config.training.block_size, + mask_times=config.training.mask_times_per_sample, + max_gen_length=config.training.max_gen_length + ) + + # Scheduler & DataLoader + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(ds) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + dl = DataLoader( + ds, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + ) + + model, optimizer, lr_scheduler, dl = accelerator.prepare(model, optimizer, lr_scheduler, dl) + + logger.info("***** Running LLaDA-V SFT training *****") + logger.info(f" Num variants = {len(ds)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + data_time_m = AverageMeter() + end = time.time() + + from tqdm.auto import tqdm + + def forward_process_batch(batch): + out = prepare_batch_noisy_embeds( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + return None + noisy_embeds, labels, p_mask, attn = out + + outputs = model.get_model()( + inputs_embeds=noisy_embeds, + attention_mask=attn, + use_cache=False, + return_dict=True, + ) + logits = model.lm_head(outputs.last_hidden_state).float() # (B, L, V) + + # logits = outputs.logits.float() # (B, L, V) + log_probs = F.log_softmax(logits, dim=-1) + + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_tok = log_probs.gather(-1, safe_labels.unsqueeze(-1)).squeeze(-1) # (B, L) + + loss_tok = - (logp_tok * p_mask).sum(dim=1) + num_mask = torch.clamp(p_mask.sum(dim=1), min=1) + loss = (loss_tok / num_mask).mean() + return loss + + first_epoch = 0 + grad_accum = config.training.gradient_accumulation_steps + + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + dl, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + # for step, batch in enumerate(progress_bar, start=1): + # data_time_m.update(time.time() - end) + + # # 关键修复:使用 Accelerate 官方推荐的梯度累加上下文管理器 + # with accelerator.accumulate(model): + # # 不需要手动除以 grad_accum,Accelerate 会在底层自动处理 + # loss = forward_process_batch(batch) + + # accelerator.backward(loss) + + # # 只有当梯度累加达到步数,需要真实更新权重时,才做梯度裁剪 + # if accelerator.sync_gradients: + # if config.training.max_grad_norm is not None: + # accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + + # optimizer.step() + # lr_scheduler.step() + # optimizer.zero_grad(set_to_none=True) + + # # 警告:坚决删掉这里的 torch.cuda.empty_cache() + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + loss = forward_process_batch(batch) + if loss is None: + continue + loss = loss / grad_accum + accelerator.backward(loss) + + if step % grad_accum == 0: + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + if step % grad_accum != 0: + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/wandb/run-20260319_223122-agxw5i56/files/diff.patch b/wandb/run-20260319_223122-agxw5i56/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..21d974113a5a5ffdb711ef8ff2de4eaa11c4b665 --- /dev/null +++ b/wandb/run-20260319_223122-agxw5i56/files/diff.patch @@ -0,0 +1,267 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..7a08a39 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,46 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [], ++ "source": [] + } + ], + "metadata": { +@@ -85,7 +125,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) diff --git a/wandb/run-20260319_223122-agxw5i56/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260319_223122-agxw5i56/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..21d974113a5a5ffdb711ef8ff2de4eaa11c4b665 --- /dev/null +++ b/wandb/run-20260319_223122-agxw5i56/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,267 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..7a08a39 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,46 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [], ++ "source": [] + } + ], + "metadata": { +@@ -85,7 +125,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) diff --git a/wandb/run-20260319_223122-agxw5i56/files/output.log b/wandb/run-20260319_223122-agxw5i56/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f6da21d229726d6431d052ba0d29de177b27514b --- /dev/null +++ b/wandb/run-20260319_223122-agxw5i56/files/output.log @@ -0,0 +1,26 @@ +03/19/2026 22:31:26 - INFO - root - Saving config to sft_lladav_video_reversal_3_19/config.yaml +Rank 0: Loaded LLaVA model: /data4/runqing/dLLM-RL/LLaDA-V +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:20<00:00, 3.37s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.97 GB Max_MA 16.97 GB CA 20.15 GB Max_CA 20 GB +CPU Virtual Memory: used = 208.51 GB, percent = 10.3% +After initializing optimizer states +MA 16.97 GB Max_MA 16.97 GB CA 20.15 GB Max_CA 20 GB +CPU Virtual Memory: used = 228.52 GB, percent = 11.3% +After initializing ZeRO optimizer +MA 16.97 GB Max_MA 16.97 GB CA 20.15 GB Max_CA 20 GB +CPU Virtual Memory: used = 229.31 GB, percent = 11.4% +03/19/2026 22:33:07 - INFO - __main__ - ***** Running LLaDA-V SFT training ***** +03/19/2026 22:33:07 - INFO - __main__ - Num variants = 95996 +03/19/2026 22:33:07 - INFO - __main__ - Num training steps = 3001 +03/19/2026 22:33:07 - INFO - __main__ - Instantaneous batch size per device = 2 +03/19/2026 22:33:07 - INFO - __main__ - Total train batch size (parallel * accum) = 32 +03/19/2026 22:33:07 - INFO - __main__ - Gradient Accumulation steps = 4 +Epoch 1/1: 0%| | 0/12000 [00:00\nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff.patchnetdisk/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff.patch189E4490C4AA826F17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a341b0ef13db33a9cfc8c700b1fd196501c0ae91a1f3805869f4c704a35346a3, Size: 11222}"} +{"time":"2026-03-19T22:31:31.801370376+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1bfe21a4d7c63fe36fba0cc6e7d8abc0349992391788b2eb567126807629b913"} +{"time":"2026-03-19T22:31:31.801479894+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txtnetdisk/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txt189E4490C4D2AC1E29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1bfe21a4d7c63fe36fba0cc6e7d8abc0349992391788b2eb567126807629b913, Size: 2966}"} +{"time":"2026-03-19T22:31:31.802152269+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=633c45789455deff40ac9ac290a8030f078715a583a3cc959d99dfeef370bdc2"} +{"time":"2026-03-19T22:31:31.802279633+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch189E4490C44A5AF84b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=633c45789455deff40ac9ac290a8030f078715a583a3cc959d99dfeef370bdc2, Size: 11222}"} +{"time":"2026-03-19T22:31:31.803630426+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f087ddbd49f5feebf121bad34dc7ee132d5e81a79967564da3bbaee6ca6044cf"} +{"time":"2026-03-19T22:31:31.803693079+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.json189E4490C4F3B71729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f087ddbd49f5feebf121bad34dc7ee132d5e81a79967564da3bbaee6ca6044cf, Size: 2650}"} +{"time":"2026-03-19T22:31:31.810924488+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e90cb55cb9fe21539c6e1e8d4e3349ca70bfaa57d266471b6485dd8c5fdb2bfe"} +{"time":"2026-03-19T22:31:31.810989568+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.pynetdisk/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.py189E4490C4D0F1F329678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/files/code/train/sft_lladav_video_reversal.py, Name: code/train/sft_lladav_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_19/agxw5i56/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260319%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260319T143131Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e90cb55cb9fe21539c6e1e8d4e3349ca70bfaa57d266471b6485dd8c5fdb2bfe, Size: 29481}"} diff --git a/wandb/run-20260319_223122-agxw5i56/logs/debug.log b/wandb/run-20260319_223122-agxw5i56/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bd3700b86c03d4737f6f4036a086a9b21b4bb47c --- /dev/null +++ b/wandb/run-20260319_223122-agxw5i56/logs/debug.log @@ -0,0 +1,20 @@ +2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_setup.py:_flush():81] Configure stats pid to 3863399 +2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/logs/debug.log +2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260319_223122-agxw5i56/logs/debug-internal.log +2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:init():844] calling init triggers +2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/sft_lladav_video_reversal.py'}} +2026-03-19 22:31:22,112 INFO MainThread:3863399 [wandb_init.py:init():892] starting backend +2026-03-19 22:31:22,373 INFO MainThread:3863399 [wandb_init.py:init():895] sending inform_init request +2026-03-19 22:31:22,377 INFO MainThread:3863399 [wandb_init.py:init():903] backend started and connected +2026-03-19 22:31:22,378 INFO MainThread:3863399 [wandb_init.py:init():973] updated telemetry +2026-03-19 22:31:22,383 INFO MainThread:3863399 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-03-19 22:31:26,722 INFO MainThread:3863399 [wandb_init.py:init():1042] starting run threads in backend +2026-03-19 22:31:26,920 INFO MainThread:3863399 [wandb_run.py:_console_start():2524] atexit reg +2026-03-19 22:31:26,920 INFO MainThread:3863399 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-03-19 22:31:26,921 INFO MainThread:3863399 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-03-19 22:31:26,921 INFO MainThread:3863399 [wandb_run.py:_redirect():2465] Redirects installed. +2026-03-19 22:31:26,928 INFO MainThread:3863399 [wandb_init.py:init():1082] run started, returning control to user process +2026-03-19 22:31:26,929 INFO MainThread:3863399 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'agxw5i56', 'experiment.project': 'sft_lladav_video_reversal_3_19', 'experiment.num_node': 1, 'experiment.logging_dir': 'sft_lladav_video_reversal_3_19/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/LLaDA-V', 'model.optimized_name': 'optimized', 'dataset.optimization_data': 'video_reversal_sft_train', 'dataset.data_type': 'video_reverse', 'training.gradient_checkpointing_enable': True, 'training.gradient_accumulation_steps': 4, 'training.batch_size_lm': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.1, 'training.upper_p': 0.9, 'training.block_size': 16, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 512, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-05, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-05, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'config': 'configs/sft_lladav_video_reversal.yaml'} diff --git a/wandb/run-20260322_031609-y30zeekk/files/code/train/sft_lladav_video_reversal.py b/wandb/run-20260322_031609-y30zeekk/files/code/train/sft_lladav_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..148964119f218130cb507d97a8a4ce672ccce54e --- /dev/null +++ b/wandb/run-20260322_031609-y30zeekk/files/code/train/sft_lladav_video_reversal.py @@ -0,0 +1,757 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator, InitProcessGroupKwargs +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +# LLaDA-V / LLaVA +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_pyav, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates +# from accelerate import DistributedDataParallelKwargs #添加 + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) # (L,) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + """ + 统一处理 Image 和 Video + 返回 (media_tensor_list, sizes, video_stats) + """ + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + # 使用官方提供的 decord 函数处理视频 + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + + # 处理成 tensor + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + # 提取 size + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + """通用媒体路径解析""" + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class SFTVariantDatasetLLADAV(Dataset): + + def __init__(self, samples: List[dict], method: str, lower: float, upper: float, block_size: int, mask_times: int, max_gen_length: int): + super().__init__() + self.samples = samples + self.method = method + self.lower = lower + self.upper = upper + self.block_size = block_size + self.mask_times = mask_times + self.max_gen_length = max_gen_length + + self.variants: List[dict] = [] + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt="\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=42) + for sid, s in enumerate(self.samples): + if self.method == "semi-ar": + step_map = s.get("step_map", None) + if step_map is None: + step_map = list(range(self.max_gen_length)) + order_list = collapse_k_unique(step_map, self.block_size) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(dict( + sample_id=sid, + prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt, + response=s["response"], + image_abs_path=s.get("image_abs_path", None), + image=s.get("image", None), + image_path=s.get("image_path", None), + video_abs_path=s.get("video_abs_path", None), + video=s.get("video", None), + video_path=s.get("video_path", None), + var_type="semi_ar", + var_param=int(st), + step_map=order_list, + mask_seed=None, + )) + elif self.method == "random_masking": + for _ in range(self.mask_times): + t = float(rng.uniform(self.lower, self.upper)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(dict( + sample_id=sid, + prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt, + response=s["response"], + image_abs_path=s.get("image_abs_path", None), + image=s.get("image", None), + image_path=s.get("image_path", None), + video_abs_path=s.get("video_abs_path", None), + video=s.get("video", None), + video_path=s.get("video_path", None), + var_type="random_masking", + var_param=t, + step_map=None, + mask_seed=mask_seed, + )) + else: + raise ValueError(f"Unknown SFT method: {self.method}") + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + v = self.variants[idx] + return { + "idx": idx, + "sample_id": v["sample_id"], + "prompt": v["prompt"], + "response": v["response"], + "image_abs_path": v.get("image_abs_path", None), + "image": v.get("image", None), + "image_path": v.get("image_path", None), + "video_abs_path": v.get("video_abs_path", None), + "video": v.get("video", None), + "video_path": v.get("video_path", None), + "var_type": v["var_type"], + "var_param": v["var_param"], + "step_map": v.get("step_map", None), + "mask_seed": v.get("mask_seed", None), + } + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "image": batch.get("image", [None])[first_pos], + "image_path": batch.get("image_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + "video": batch.get("video", [None])[first_pos], + "video_path": batch.get("video_path", [None])[first_pos], + } + + # 优先检测视频,其次检测图片 + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + + modalities = [media_type] + + # 若包含视频且需附加时间指引,将参数注入 prompt + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + # 剔除原有可能存在的图片 Token 以免重叠 + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + # Tokenize (内部会自动 prepend DEFAULT_IMAGE_TOKEN) + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) # (L, D) + new_labels = new_labels.squeeze(0) # (L,) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) # (M,) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [] + for i in range(bsz): + sid = int(batch["sample_id"][i]) + lengths.append(int(sample_cache[sid]["clean_embeds"].shape[0])) + Lmax = max(lengths) + D = next(iter(sample_cache.values()))["clean_embeds"].shape[1] + + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list = [] + labels_list = [] + p_mask_list = [] + attn_list = [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + step_map = batch["step_map"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + + if var_type == "semi_ar": + order = torch.as_tensor(step_map, device=device) + M = asst_idx.numel() + if order.numel() != M: + m = min(M, order.numel()) + asst_idx = asst_idx[:m] + order = order[:m] + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx] = order + step_val = int(var_param) + + p = torch.empty(L, device=device).uniform_(0.0, 1.0) + lower = 0.0 + 1e-6 + upper = 1.0 - 1e-6 + thresh = (upper - lower) * torch.rand(1, device=device) + lower + block_mask = (order_full == step_val) & (p < thresh) + + mask_pos = (order_full > step_val) | block_mask + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + pmask = block_mask + + elif var_type == "random_masking": + M = asst_idx.numel() + if M == 0: + noisy_embeds = clean_embeds + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + gen = torch.Generator(device="cpu") + gen.manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + rnd = rnd_cpu.to(device=device) + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + else: + continue + + if not pmask.any(): + if asst_idx.numel() > 0: + continue + else: + print("*********bad data*********") + print(batch["sample_id"][i]) + continue + + # if not pmask.any(): + # print('*****no token masked*****') + # if asst_idx.numel() > 0: + # # 强制 Mask 最后一个 assistant token + # force_idx = asst_idx[-1] + # pmask[force_idx] = True + # noisy_embeds[force_idx] = mask_embed + # else: + # print("*********bad data*********") + # print(batch["sample_id"][i]) + # # 如果连 assistant token 都没有,那这条数据本身就是坏的,只能跳过 + # continue + + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + return noisy_batch, labels_batch, p_mask_batch, attn_batch + +# ... 余下 save_checkpoint 和 main() 训练循环逻辑无需修改,保持原样即可 ... +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + # TF32 + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + + # 🌟 实例化 DDP 处理器,允许存在未使用的参数 + # ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True) + # process_group_kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=3600)) + + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with="wandb", + project_dir=config.experiment.logging_dir, + split_batches=True, + # kwargs_handlers=[process_group_kwargs] # 🌟 把超时设置传进去 + # kwargs_handlers=[ddp_kwargs], # <-- 把这个参数传进去 + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + pretrained_model = config.model.pretrained_model + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + with open("./data/" + config.dataset.optimization_data + ".json", 'r') as f: + data_list = json.load(f) + + ds = SFTVariantDatasetLLADAV( + samples=data_list, + method=config.training.method, + lower=config.training.lower_p, + upper=config.training.upper_p, + block_size=config.training.block_size, + mask_times=config.training.mask_times_per_sample, + max_gen_length=config.training.max_gen_length + ) + + # Scheduler & DataLoader + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(ds) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + dl = DataLoader( + ds, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + ) + + model, optimizer, lr_scheduler, dl = accelerator.prepare(model, optimizer, lr_scheduler, dl) + + logger.info("***** Running LLaDA-V SFT training *****") + logger.info(f" Num variants = {len(ds)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + data_time_m = AverageMeter() + end = time.time() + + from tqdm.auto import tqdm + + def forward_process_batch(batch): + out = prepare_batch_noisy_embeds( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + return None + noisy_embeds, labels, p_mask, attn = out + + outputs = model.get_model()( + inputs_embeds=noisy_embeds, + attention_mask=attn, + use_cache=False, + return_dict=True, + ) + logits = model.lm_head(outputs.last_hidden_state).float() # (B, L, V) + + # logits = outputs.logits.float() # (B, L, V) + log_probs = F.log_softmax(logits, dim=-1) + + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_tok = log_probs.gather(-1, safe_labels.unsqueeze(-1)).squeeze(-1) # (B, L) + + loss_tok = - (logp_tok * p_mask).sum(dim=1) + num_mask = torch.clamp(p_mask.sum(dim=1), min=1) + loss = (loss_tok / num_mask).mean() + return loss + + first_epoch = 0 + grad_accum = config.training.gradient_accumulation_steps + + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + dl, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + # for step, batch in enumerate(progress_bar, start=1): + # data_time_m.update(time.time() - end) + + # # 关键修复:使用 Accelerate 官方推荐的梯度累加上下文管理器 + # with accelerator.accumulate(model): + # # 不需要手动除以 grad_accum,Accelerate 会在底层自动处理 + # loss = forward_process_batch(batch) + + # accelerator.backward(loss) + + # # 只有当梯度累加达到步数,需要真实更新权重时,才做梯度裁剪 + # if accelerator.sync_gradients: + # if config.training.max_grad_norm is not None: + # accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + + # optimizer.step() + # lr_scheduler.step() + # optimizer.zero_grad(set_to_none=True) + + # # 警告:坚决删掉这里的 torch.cuda.empty_cache() + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + loss = forward_process_batch(batch) + if loss is None: + continue + loss = loss / grad_accum + accelerator.backward(loss) + + if step % grad_accum == 0: + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + if step % grad_accum != 0: + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/wandb/run-20260322_031609-y30zeekk/files/diff.patch b/wandb/run-20260322_031609-y30zeekk/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1cab8584a841e0c8b60d60d929914b3ad3866922 --- /dev/null +++ b/wandb/run-20260322_031609-y30zeekk/files/diff.patch @@ -0,0 +1,270 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..5701c74 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,62 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +141,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) diff --git a/wandb/run-20260322_031609-y30zeekk/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260322_031609-y30zeekk/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1cab8584a841e0c8b60d60d929914b3ad3866922 --- /dev/null +++ b/wandb/run-20260322_031609-y30zeekk/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,270 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..5701c74 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,62 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +141,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) diff --git a/wandb/run-20260322_031609-y30zeekk/files/output.log b/wandb/run-20260322_031609-y30zeekk/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8871233ced0cfb4b4a2aa68443f52f8902b56f51 --- /dev/null +++ b/wandb/run-20260322_031609-y30zeekk/files/output.log @@ -0,0 +1,26 @@ +03/22/2026 03:16:15 - INFO - root - Saving config to sft_lladav_video_reversal_3_21/config.yaml +Rank 0: Loaded LLaVA model: /data4/runqing/dLLM-RL/LLaDA-V +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 6/6 [00:25<00:00, 4.19s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.97 GB Max_MA 16.97 GB CA 17.23 GB Max_CA 17 GB +CPU Virtual Memory: used = 133.3 GB, percent = 6.6% +After initializing optimizer states +MA 16.97 GB Max_MA 16.97 GB CA 17.23 GB Max_CA 17 GB +CPU Virtual Memory: used = 158.85 GB, percent = 7.9% +After initializing ZeRO optimizer +MA 16.97 GB Max_MA 16.97 GB CA 17.23 GB Max_CA 17 GB +CPU Virtual Memory: used = 160.29 GB, percent = 8.0% +03/22/2026 03:18:02 - INFO - __main__ - ***** Running LLaDA-V SFT training ***** +03/22/2026 03:18:02 - INFO - __main__ - Num variants = 39024 +03/22/2026 03:18:02 - INFO - __main__ - Num training steps = 611 +03/22/2026 03:18:02 - INFO - __main__ - Instantaneous batch size per device = 2 +03/22/2026 03:18:02 - INFO - __main__ - Total train batch size (parallel * accum) = 64 +03/22/2026 03:18:02 - INFO - __main__ - Gradient Accumulation steps = 4 +Epoch 1/1: 0%| | 0/2439 [00:00\nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff.patchnetdisk/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff.patch189EF14496B0C68629678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=543a564ab27a29e0506e5496b121bfa5ec3df952fb270dc6cbc8e792933f1a0d, Size: 11427}"} +{"time":"2026-03-22T03:16:20.123748872+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=65fe9536dca78a54fcb1be38cc2071ac77844ac78ae053d4f2dda5671108cda7"} +{"time":"2026-03-22T03:16:20.123818956+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txtnetdisk/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txt189EF14496BBA83329678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=65fe9536dca78a54fcb1be38cc2071ac77844ac78ae053d4f2dda5671108cda7, Size: 2966}"} +{"time":"2026-03-22T03:16:20.124069975+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=76db83e542423b361e9411c108e0835fa100fe9691efcbc1f98e5e3a28fedecf"} +{"time":"2026-03-22T03:16:20.124180799+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.json189EF14496AFDED44b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=76db83e542423b361e9411c108e0835fa100fe9691efcbc1f98e5e3a28fedecf, Size: 2650}"} +{"time":"2026-03-22T03:16:20.126422985+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8f5ee1f91644676947fa6db83d772470d8ed9fdcf739aa7e905ac62cb16f53d"} +{"time":"2026-03-22T03:16:20.126566466+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch189EF14496C3432417e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8f5ee1f91644676947fa6db83d772470d8ed9fdcf739aa7e905ac62cb16f53d, Size: 11427}"} +{"time":"2026-03-22T03:16:20.13421156+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=59e9f74c84de8a143e6c6b9f2aba14af539692b331769d7512155279863bd62c"} +{"time":"2026-03-22T03:16:20.134366382+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.pynetdisk/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.py189EF14496CC1BA517e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/files/code/train/sft_lladav_video_reversal.py, Name: code/train/sft_lladav_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lladav_video_reversal_3_21/y30zeekk/code/train/sft_lladav_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260321%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260321T191620Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=59e9f74c84de8a143e6c6b9f2aba14af539692b331769d7512155279863bd62c, Size: 29865}"} diff --git a/wandb/run-20260322_031609-y30zeekk/logs/debug.log b/wandb/run-20260322_031609-y30zeekk/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4f4ade9cc346e69ed81eb696c8851627d182bbbe --- /dev/null +++ b/wandb/run-20260322_031609-y30zeekk/logs/debug.log @@ -0,0 +1,20 @@ +2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_setup.py:_flush():81] Configure stats pid to 917982 +2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/logs/debug.log +2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260322_031609-y30zeekk/logs/debug-internal.log +2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:init():844] calling init triggers +2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/sft_lladav_video_reversal.py'}} +2026-03-22 03:16:09,679 INFO MainThread:917982 [wandb_init.py:init():892] starting backend +2026-03-22 03:16:09,925 INFO MainThread:917982 [wandb_init.py:init():895] sending inform_init request +2026-03-22 03:16:09,928 INFO MainThread:917982 [wandb_init.py:init():903] backend started and connected +2026-03-22 03:16:09,929 INFO MainThread:917982 [wandb_init.py:init():973] updated telemetry +2026-03-22 03:16:09,935 INFO MainThread:917982 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-03-22 03:16:15,046 INFO MainThread:917982 [wandb_init.py:init():1042] starting run threads in backend +2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_console_start():2524] atexit reg +2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-03-22 03:16:15,154 INFO MainThread:917982 [wandb_run.py:_redirect():2465] Redirects installed. +2026-03-22 03:16:15,157 INFO MainThread:917982 [wandb_init.py:init():1082] run started, returning control to user process +2026-03-22 03:16:15,159 INFO MainThread:917982 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'y30zeekk', 'experiment.project': 'sft_lladav_video_reversal_3_21', 'experiment.num_node': 1, 'experiment.logging_dir': 'sft_lladav_video_reversal_3_21/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/LLaDA-V', 'model.optimized_name': 'optimized', 'dataset.optimization_data': 'video_reversal_sft_train_cleaned', 'dataset.data_type': 'video_reverse', 'training.gradient_checkpointing_enable': True, 'training.gradient_accumulation_steps': 4, 'training.batch_size_lm': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.1, 'training.upper_p': 0.9, 'training.block_size': 16, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 512, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-05, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-05, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'config': 'configs/sft_lladav_video_reversal.yaml'} diff --git a/wandb/run-20260322_031609-y30zeekk/run-y30zeekk.wandb b/wandb/run-20260322_031609-y30zeekk/run-y30zeekk.wandb new file mode 100644 index 0000000000000000000000000000000000000000..418a8e498507d3769cbf4a52e71b9a0dd1423c7c Binary files /dev/null and b/wandb/run-20260322_031609-y30zeekk/run-y30zeekk.wandb differ diff --git a/wandb/run-20260323_031014-fmwqi7oc/files/code/train/rl_lladav.py b/wandb/run-20260323_031014-fmwqi7oc/files/code/train/rl_lladav.py new file mode 100644 index 0000000000000000000000000000000000000000..f9b5a004b7e8420453330994c209224cc328f90e --- /dev/null +++ b/wandb/run-20260323_031014-fmwqi7oc/files/code/train/rl_lladav.py @@ -0,0 +1,806 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 0) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", False) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LLaDA-V model and tokenizer") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wandb/run-20260323_031014-fmwqi7oc/files/config.yaml b/wandb/run-20260323_031014-fmwqi7oc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9864ab90565991638b7b3e8ffcb3d919da508dc --- /dev/null +++ b/wandb/run-20260323_031014-fmwqi7oc/files/config.yaml @@ -0,0 +1,279 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lladav.py + e: + 55pvqnbadfj4j7x2fc9exogdpb6eojxo: + args: + - config=configs/rl_lladav_video_reversal.yaml + - experiment.current_epoch=1 + codePath: train/rl_lladav.py + codePathLocal: train/rl_lladav.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.4" + disk: + /: + total: "806578724864" + used: "571723153408" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113813504" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lladav.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-03-22T19:10:14.663079Z" + writerId: 55pvqnbadfj4j7x2fc9exogdpb6eojxo + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 13 + - 14 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lladav_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_train +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: true +evaluation.max_gen_length: + value: 256 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 256 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 1 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2_rl +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lladav_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lladav_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 160 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lladav +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 2 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 64 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmwqi7oc diff --git a/wandb/run-20260323_031014-fmwqi7oc/files/diff.patch b/wandb/run-20260323_031014-fmwqi7oc/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1cab8584a841e0c8b60d60d929914b3ad3866922 --- /dev/null +++ b/wandb/run-20260323_031014-fmwqi7oc/files/diff.patch @@ -0,0 +1,270 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..5701c74 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,62 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +141,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) diff --git a/wandb/run-20260323_031014-fmwqi7oc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260323_031014-fmwqi7oc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1cab8584a841e0c8b60d60d929914b3ad3866922 --- /dev/null +++ b/wandb/run-20260323_031014-fmwqi7oc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,270 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..5701c74 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,62 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_train.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +141,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) diff --git a/wandb/run-20260323_031014-fmwqi7oc/files/output.log b/wandb/run-20260323_031014-fmwqi7oc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..895ede6a4b9497a8d59451e95bf4eb322e0f685d --- /dev/null +++ b/wandb/run-20260323_031014-fmwqi7oc/files/output.log @@ -0,0 +1,82 @@ +03/23/2026 03:10:19 - INFO - root - Saving config to rl_lladav_video_reversal/config.yaml +03/23/2026 03:10:19 - INFO - __main__ - Loading LLaDA-V model and tokenizer +Rank 0: Loaded LLaVA model: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:22<00:00, 5.68s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +03/23/2026 03:10:45 - INFO - __main__ - Preprocessing Data +03/23/2026 03:10:45 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 160.24 GB, percent = 8.0% +After initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 186.43 GB, percent = 9.2% +After initializing ZeRO optimizer +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 189.08 GB, percent = 9.4% +03/23/2026 03:12:00 - INFO - __main__ - ***** Running inference ***** +03/23/2026 03:12:00 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +03/23/2026 03:14:03 - INFO - __main__ - ***** Running training ***** +03/23/2026 03:14:03 - INFO - __main__ - Num variants = 384 +03/23/2026 03:14:03 - INFO - __main__ - Num training steps = 2 +03/23/2026 03:14:03 - INFO - __main__ - Instantaneous batch size per device = 2 +03/23/2026 03:14:03 - INFO - __main__ - Total train batch size (parallel * accum) = 1024 +03/23/2026 03:14:03 - INFO - __main__ - Gradient Accumulation steps = 64 +start rl training... +Epoch 1/1: 8%|▊ | 2/24 [00:32<06:02, 16.47s/it] +Traceback (most recent call last): + File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 806, in + main() + File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 767, in main + loss_lm = forward_process( + File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 692, in forward_process + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 1027, in forward + layer_outputs = decoder_layer( + File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 759, in forward + hidden_states = self.post_attention_layernorm(hidden_states) + File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 89, in forward + variance = hidden_states.pow(2).mean(-1, keepdim=True) +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 214.00 MiB. GPU 0 has a total capacity of 79.11 GiB of which 208.81 MiB is free. Process 2750357 has 78.90 GiB memory in use. Of the allocated memory 73.41 GiB is allocated by PyTorch, and 1.34 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank0]: Traceback (most recent call last): +[rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 806, in +[rank0]: main() +[rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 767, in main +[rank0]: loss_lm = forward_process( +[rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 692, in forward_process +[rank0]: outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) +[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 1027, in forward +[rank0]: layer_outputs = decoder_layer( +[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 759, in forward +[rank0]: hidden_states = self.post_attention_layernorm(hidden_states) +[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/data4/runqing/dLLM-RL/train/llava/model/language_model/modeling_llada.py", line 89, in forward +[rank0]: variance = hidden_states.pow(2).mean(-1, keepdim=True) +[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 214.00 MiB. GPU 0 has a total capacity of 79.11 GiB of which 208.81 MiB is free. Process 2750357 has 78.90 GiB memory in use. Of the allocated memory 73.41 GiB is allocated by PyTorch, and 1.34 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) diff --git a/wandb/run-20260323_031014-fmwqi7oc/files/requirements.txt b/wandb/run-20260323_031014-fmwqi7oc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53a0646712bb7ab0c9a25728e3226aa2d67dafd0 --- /dev/null +++ b/wandb/run-20260323_031014-fmwqi7oc/files/requirements.txt @@ -0,0 +1,156 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260323_031014-fmwqi7oc/files/wandb-metadata.json b/wandb/run-20260323_031014-fmwqi7oc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..328a4b39bb0df8169771c70230c48c83aa9e7487 --- /dev/null +++ b/wandb/run-20260323_031014-fmwqi7oc/files/wandb-metadata.json @@ -0,0 +1,93 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-03-22T19:10:14.663079Z", + "args": [ + "config=configs/rl_lladav_video_reversal.yaml", + "experiment.current_epoch=1" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py", + "codePath": "train/rl_lladav.py", + "codePathLocal": "train/rl_lladav.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "571723153408" + } + }, + "memory": { + "total": "2164113813504" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.4", + "writerId": "55pvqnbadfj4j7x2fc9exogdpb6eojxo" +} \ No newline at end of file diff --git a/wandb/run-20260323_031014-fmwqi7oc/files/wandb-summary.json b/wandb/run-20260323_031014-fmwqi7oc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9c6826bba8358ca57cae8e6b2912266bc9e674ef --- /dev/null +++ b/wandb/run-20260323_031014-fmwqi7oc/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":256},"_runtime":256} \ No newline at end of file diff --git a/wandb/run-20260323_031014-fmwqi7oc/logs/debug-core.log b/wandb/run-20260323_031014-fmwqi7oc/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..e8cae5af393821f613067f6b3b0da51fa05ae467 --- /dev/null +++ b/wandb/run-20260323_031014-fmwqi7oc/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-03-23T03:10:14.734689708+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpxva6clnc/port-3523574.txt","pid":3523574,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-23T03:10:14.735635443+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3523574} +{"time":"2026-03-23T03:10:14.735628068+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3523574-3524049-679808348/socket","Net":"unix"}} +{"time":"2026-03-23T03:10:14.915970148+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-23T03:10:14.923903515+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmwqi7oc","id":"1(@)"} +{"time":"2026-03-23T03:10:19.250521705+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmwqi7oc","id":"1(@)"} +{"time":"2026-03-23T03:10:24.490584047+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"srafqf41avyr"} +{"time":"2026-03-23T03:14:36.354854515+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-03-23T03:14:36.354904386+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-03-23T03:14:36.354956667+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-03-23T03:14:36.354918741+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-03-23T03:14:36.355031381+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3523574-3524049-679808348/socket","Net":"unix"}} +{"time":"2026-03-23T03:14:36.574170314+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-03-23T03:14:36.574226901+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-03-23T03:14:36.574272235+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260323_031014-fmwqi7oc/logs/debug-internal.log b/wandb/run-20260323_031014-fmwqi7oc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d7a1c5f20ee14c224ddf84c3ed11376e74835942 --- /dev/null +++ b/wandb/run-20260323_031014-fmwqi7oc/logs/debug-internal.log @@ -0,0 +1,29 @@ +{"time":"2026-03-23T03:10:14.923991754+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-23T03:10:19.249985082+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-03-23T03:10:19.250206083+08:00","level":"INFO","msg":"stream: created new stream","id":"fmwqi7oc"} +{"time":"2026-03-23T03:10:19.250314544+08:00","level":"INFO","msg":"handler: started","stream_id":"fmwqi7oc"} +{"time":"2026-03-23T03:10:19.250506873+08:00","level":"INFO","msg":"stream: started","id":"fmwqi7oc"} +{"time":"2026-03-23T03:10:19.250555978+08:00","level":"INFO","msg":"writer: started","stream_id":"fmwqi7oc"} +{"time":"2026-03-23T03:10:19.250551745+08:00","level":"INFO","msg":"sender: started","stream_id":"fmwqi7oc"} +{"time":"2026-03-23T03:10:19.253219433+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-03-23T03:10:24.455218043+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=424ac51772e4e6aeadd1916a4c5be0ccd13e3e923cbb73fc604d2577f3cda919"} +{"time":"2026-03-23T03:10:24.455395899+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=810d6278f7888329964e292ae4b8d352346389dba5b1e0d16660a92138f8fa0b"} +{"time":"2026-03-23T03:10:24.455410759+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.json189F3F86588581D517e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=424ac51772e4e6aeadd1916a4c5be0ccd13e3e923cbb73fc604d2577f3cda919, Size: 2635}"} +{"time":"2026-03-23T03:10:24.455507161+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txt189F3F8658979FBE17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=810d6278f7888329964e292ae4b8d352346389dba5b1e0d16660a92138f8fa0b, Size: 2966}"} +{"time":"2026-03-23T03:10:24.455771816+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8742bb45572bf89449139797076cdb743a0f0e84b1b2d691043a6f8cab81636"} +{"time":"2026-03-23T03:10:24.455883557+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch189F3F8658A48B884b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8742bb45572bf89449139797076cdb743a0f0e84b1b2d691043a6f8cab81636, Size: 11427}"} +{"time":"2026-03-23T03:10:24.458949814+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c0d60641ef66f42e5ff9c0558786e2031e714aa06ed1c87f70bfd6432a5e2bce"} +{"time":"2026-03-23T03:10:24.459156615+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patch189F3F8658B037044b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c0d60641ef66f42e5ff9c0558786e2031e714aa06ed1c87f70bfd6432a5e2bce, Size: 11427}"} +{"time":"2026-03-23T03:10:24.460144111+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b854582f73a02708bfcd8010e9db9ec37b13d12c64988b1b7d8d7a37414bc0e1"} +{"time":"2026-03-23T03:10:24.460213159+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.pynetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.py189F3F865877B3DB29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191024Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b854582f73a02708bfcd8010e9db9ec37b13d12c64988b1b7d8d7a37414bc0e1, Size: 31804}"} +{"time":"2026-03-23T03:14:36.354902346+08:00","level":"INFO","msg":"stream: closing","id":"fmwqi7oc"} +{"time":"2026-03-23T03:14:36.433198106+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=884fe6b1bfe554b7c975118d664e5b487c0bf178608ab506bd9a0e45c348c413"} +{"time":"2026-03-23T03:14:36.433310818+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.lognetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.log189F3FC103A8B9D529678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=884fe6b1bfe554b7c975118d664e5b487c0bf178608ab506bd9a0e45c348c413, Size: 7319}"} +{"time":"2026-03-23T03:14:36.437455015+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ba2517cac97729d5e42d029ad94e53f0fdce01e5c8382b91f9f83fad1535086d"} +{"time":"2026-03-23T03:14:36.437526478+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.json189F3FC103D3486C17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ba2517cac97729d5e42d029ad94e53f0fdce01e5c8382b91f9f83fad1535086d, Size: 41}"} +{"time":"2026-03-23T03:14:36.437757914+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a8c9b732a52ea76e7908bfd3e2f5b5536c95c03abe7dc4d1913e42c397ea578d"} +{"time":"2026-03-23T03:14:36.437912275+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yamlnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yaml189F3FC103DD384029678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/fmwqi7oc/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260322%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260322T191436Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a8c9b732a52ea76e7908bfd3e2f5b5536c95c03abe7dc4d1913e42c397ea578d, Size: 7766}"} +{"time":"2026-03-23T03:14:36.438391389+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-03-23T03:14:36.573284944+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmwqi7oc"} +{"time":"2026-03-23T03:14:36.573493717+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmwqi7oc"} +{"time":"2026-03-23T03:14:36.573524005+08:00","level":"INFO","msg":"stream: closed","id":"fmwqi7oc"} diff --git a/wandb/run-20260323_031014-fmwqi7oc/logs/debug.log b/wandb/run-20260323_031014-fmwqi7oc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7fccfd68e7d4d6cd91e1f88bc8edb6bed063330e --- /dev/null +++ b/wandb/run-20260323_031014-fmwqi7oc/logs/debug.log @@ -0,0 +1,22 @@ +2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_setup.py:_flush():81] Configure stats pid to 3523574 +2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/logs/debug.log +2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260323_031014-fmwqi7oc/logs/debug-internal.log +2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:init():844] calling init triggers +2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-03-23 03:10:14,664 INFO MainThread:3523574 [wandb_init.py:init():892] starting backend +2026-03-23 03:10:14,916 INFO MainThread:3523574 [wandb_init.py:init():895] sending inform_init request +2026-03-23 03:10:14,920 INFO MainThread:3523574 [wandb_init.py:init():903] backend started and connected +2026-03-23 03:10:14,923 INFO MainThread:3523574 [wandb_init.py:init():973] updated telemetry +2026-03-23 03:10:14,928 INFO MainThread:3523574 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-03-23 03:10:19,377 INFO MainThread:3523574 [wandb_init.py:init():1042] starting run threads in backend +2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_console_start():2524] atexit reg +2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-03-23 03:10:19,485 INFO MainThread:3523574 [wandb_run.py:_redirect():2465] Redirects installed. +2026-03-23 03:10:19,489 INFO MainThread:3523574 [wandb_init.py:init():1082] run started, returning control to user process +2026-03-23 03:10:19,490 INFO MainThread:3523574 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmwqi7oc', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 160, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 1, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 64, 'training.batch_size_lm': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': True, 'evaluation.eval_dataset': 'video_reversal_rl_train', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 256, 'evaluation.max_gen_length': 256, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal.yaml'} +2026-03-23 03:14:36,355 INFO wandb-AsyncioManager-main:3523574 [service_client.py:_forward_responses():134] Reached EOF. +2026-03-23 03:14:36,355 INFO wandb-AsyncioManager-main:3523574 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/wandb/run-20260323_031014-fmwqi7oc/run-fmwqi7oc.wandb b/wandb/run-20260323_031014-fmwqi7oc/run-fmwqi7oc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1450dd2f6c98fcbe284d6b0d65895a2f84b52074 Binary files /dev/null and b/wandb/run-20260323_031014-fmwqi7oc/run-fmwqi7oc.wandb differ diff --git a/wandb/run-20260324_164425-eb44q2tb/files/code/train/rl_lladav.py b/wandb/run-20260324_164425-eb44q2tb/files/code/train/rl_lladav.py new file mode 100644 index 0000000000000000000000000000000000000000..b9474c8df83a0d8e9b63265b3381af73f495c402 --- /dev/null +++ b/wandb/run-20260324_164425-eb44q2tb/files/code/train/rl_lladav.py @@ -0,0 +1,815 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LLaDA-V model and tokenizer") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + accelerator.log({ + "train/total_loss": loss_lm.item() * grad_accum, + "train/policy_loss": p_loss.item(), + "train/kl_loss": k_loss.item(), + "train/avg_ratio": avg_ratio.item(), + "train/learning_rate": current_lr, + "train/epoch": epoch, + }) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wandb/run-20260324_164425-eb44q2tb/files/config.yaml b/wandb/run-20260324_164425-eb44q2tb/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3fd976313b9d651b8ced8e1e305ec456df7efd7e --- /dev/null +++ b/wandb/run-20260324_164425-eb44q2tb/files/config.yaml @@ -0,0 +1,279 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lladav.py + e: + sr0a7mkap2eiuw8cpbm7o5idgj2l6pki: + args: + - config=configs/rl_lladav_video_reversal.yaml + - experiment.current_epoch=15 + codePath: train/rl_lladav.py + codePathLocal: train/rl_lladav.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.4" + disk: + /: + total: "806578724864" + used: "572337045504" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113813504" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lladav.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-03-24T08:44:25.714950Z" + writerId: sr0a7mkap2eiuw8cpbm7o5idgj2l6pki + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 13 + - 14 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lladav_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: true +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 15 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2_rl +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lladav_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lladav_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 20 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lladav +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 64 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: eb44q2tb diff --git a/wandb/run-20260324_164425-eb44q2tb/files/diff.patch b/wandb/run-20260324_164425-eb44q2tb/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260324_164425-eb44q2tb/files/diff.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260324_164425-eb44q2tb/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260324_164425-eb44q2tb/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260324_164425-eb44q2tb/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260324_164425-eb44q2tb/files/output.log b/wandb/run-20260324_164425-eb44q2tb/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1f923ca452513e10402014bf7b982bf1df84a026 --- /dev/null +++ b/wandb/run-20260324_164425-eb44q2tb/files/output.log @@ -0,0 +1,42 @@ +03/24/2026 16:44:30 - INFO - root - Saving config to rl_lladav_video_reversal/config.yaml +03/24/2026 16:44:30 - INFO - __main__ - Loading LLaDA-V model and tokenizer +Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal/ckpt/optimized +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.39s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +03/24/2026 16:44:52 - INFO - __main__ - Preprocessing Data +03/24/2026 16:44:52 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 178.31 GB, percent = 8.8% +After initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 200.78 GB, percent = 10.0% +After initializing ZeRO optimizer +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 202.31 GB, percent = 10.0% +03/24/2026 16:46:09 - INFO - __main__ - ***** Running inference ***** +03/24/2026 16:46:09 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +03/24/2026 16:50:40 - INFO - __main__ - ***** Running training ***** +03/24/2026 16:50:40 - INFO - __main__ - Num variants = 704 +03/24/2026 16:50:40 - INFO - __main__ - Num training steps = 3 +03/24/2026 16:50:40 - INFO - __main__ - Instantaneous batch size per device = 1 +03/24/2026 16:50:40 - INFO - __main__ - Total train batch size (parallel * accum) = 512 +03/24/2026 16:50:40 - INFO - __main__ - Gradient Accumulation steps = 64 +start rl training... +Epoch 1/1: 72%|███████▏ | 63/88 [11:05<04:24, 10.56s/it] +Traceback (most recent call last): + File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 815, in + main() + File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 782, in main + "train/policy_loss": p_loss.item(), +NameError: name 'p_loss' is not defined +[rank0]: Traceback (most recent call last): +[rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 815, in +[rank0]: main() +[rank0]: File "/data4/runqing/dLLM-RL/train/rl_lladav.py", line 782, in main +[rank0]: "train/policy_loss": p_loss.item(), +[rank0]: NameError: name 'p_loss' is not defined diff --git a/wandb/run-20260324_164425-eb44q2tb/files/requirements.txt b/wandb/run-20260324_164425-eb44q2tb/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53a0646712bb7ab0c9a25728e3226aa2d67dafd0 --- /dev/null +++ b/wandb/run-20260324_164425-eb44q2tb/files/requirements.txt @@ -0,0 +1,156 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260324_164425-eb44q2tb/files/wandb-metadata.json b/wandb/run-20260324_164425-eb44q2tb/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..80d91de65175ff4c7e915c61c0b30e3b8d1e760b --- /dev/null +++ b/wandb/run-20260324_164425-eb44q2tb/files/wandb-metadata.json @@ -0,0 +1,93 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-03-24T08:44:25.714950Z", + "args": [ + "config=configs/rl_lladav_video_reversal.yaml", + "experiment.current_epoch=15" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py", + "codePath": "train/rl_lladav.py", + "codePathLocal": "train/rl_lladav.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "572337045504" + } + }, + "memory": { + "total": "2164113813504" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.4", + "writerId": "sr0a7mkap2eiuw8cpbm7o5idgj2l6pki" +} \ No newline at end of file diff --git a/wandb/run-20260324_164425-eb44q2tb/files/wandb-summary.json b/wandb/run-20260324_164425-eb44q2tb/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0e1f89c24bbe3488dfcb0f004b5ae009d609cb6d --- /dev/null +++ b/wandb/run-20260324_164425-eb44q2tb/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":1036,"_wandb":{"runtime":1036}} \ No newline at end of file diff --git a/wandb/run-20260324_164425-eb44q2tb/logs/debug-core.log b/wandb/run-20260324_164425-eb44q2tb/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..6c02cc89e23a9187b0c9565645b4587d71508317 --- /dev/null +++ b/wandb/run-20260324_164425-eb44q2tb/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-03-24T16:44:25.780084766+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_w7chab5/port-1691523.txt","pid":1691523,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-24T16:44:25.780884873+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1691523} +{"time":"2026-03-24T16:44:25.780852026+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1691523-1692232-1524229333/socket","Net":"unix"}} +{"time":"2026-03-24T16:44:25.961716708+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-24T16:44:25.970032581+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"eb44q2tb","id":"1(@)"} +{"time":"2026-03-24T16:44:30.275637457+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"eb44q2tb","id":"1(@)"} +{"time":"2026-03-24T16:44:35.492970552+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"pfjk4x7e02ih"} +{"time":"2026-03-24T17:01:46.448086485+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-03-24T17:01:46.448222721+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-03-24T17:01:46.448192503+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-03-24T17:01:46.448403899+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-03-24T17:01:46.448442769+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1691523-1692232-1524229333/socket","Net":"unix"}} +{"time":"2026-03-24T17:01:46.588524788+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-03-24T17:01:46.588559276+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-03-24T17:01:46.588574496+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260324_164425-eb44q2tb/logs/debug-internal.log b/wandb/run-20260324_164425-eb44q2tb/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9c5de9277fab94efc7548626e91a8a73b20e1822 --- /dev/null +++ b/wandb/run-20260324_164425-eb44q2tb/logs/debug-internal.log @@ -0,0 +1,29 @@ +{"time":"2026-03-24T16:44:25.970209487+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-24T16:44:30.272794359+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-03-24T16:44:30.274781439+08:00","level":"INFO","msg":"stream: created new stream","id":"eb44q2tb"} +{"time":"2026-03-24T16:44:30.274957975+08:00","level":"INFO","msg":"handler: started","stream_id":"eb44q2tb"} +{"time":"2026-03-24T16:44:30.275605838+08:00","level":"INFO","msg":"writer: started","stream_id":"eb44q2tb"} +{"time":"2026-03-24T16:44:30.275607151+08:00","level":"INFO","msg":"stream: started","id":"eb44q2tb"} +{"time":"2026-03-24T16:44:30.275615082+08:00","level":"INFO","msg":"sender: started","stream_id":"eb44q2tb"} +{"time":"2026-03-24T16:44:30.27762458+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-03-24T16:44:35.521989789+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8b3af0f9bd2bc9a1d25e7c95f703d6e7f082c92a5ddeaa4bb217a9b7cfe80832"} +{"time":"2026-03-24T16:44:35.522055528+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=aaee6c8449b2ae1a780ad0c20f3b6210d18b6720e515f80b6266904d04c19233"} +{"time":"2026-03-24T16:44:35.522099395+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch189FBA88EFF948C54b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8b3af0f9bd2bc9a1d25e7c95f703d6e7f082c92a5ddeaa4bb217a9b7cfe80832, Size: 12686}"} +{"time":"2026-03-24T16:44:35.522145559+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.json189FBA88F07C428E29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=aaee6c8449b2ae1a780ad0c20f3b6210d18b6720e515f80b6266904d04c19233, Size: 2636}"} +{"time":"2026-03-24T16:44:35.524085548+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=638a1b72719105e2fa963dfad98344f6b9593e0368c367c1724abafa9f17d52a"} +{"time":"2026-03-24T16:44:35.52450294+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3d633c46405ba4ce40f9454aeca31c83c8d300c6eae3823e88f75b255c88ec8a"} +{"time":"2026-03-24T16:44:35.525507802+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patch189FBA88F08CB76A29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=638a1b72719105e2fa963dfad98344f6b9593e0368c367c1724abafa9f17d52a, Size: 12686}"} +{"time":"2026-03-24T16:44:35.52555316+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txt189FBA88F01C4C5D4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3d633c46405ba4ce40f9454aeca31c83c8d300c6eae3823e88f75b255c88ec8a, Size: 2966}"} +{"time":"2026-03-24T16:44:35.533015823+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4421e08beab520f2a67729be80eff77a1e13d1128016b44b9b31a9397524de55"} +{"time":"2026-03-24T16:44:35.533064379+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.pynetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.py189FBA88F09E62CB29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T084435Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4421e08beab520f2a67729be80eff77a1e13d1128016b44b9b31a9397524de55, Size: 32189}"} +{"time":"2026-03-24T17:01:46.448220293+08:00","level":"INFO","msg":"stream: closing","id":"eb44q2tb"} +{"time":"2026-03-24T17:01:46.543269717+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e3cd94ab34d1b5cb38d2d62928042acf4a195aacb875e5b3ce61af7e133f5d6c"} +{"time":"2026-03-24T17:01:46.543380827+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.lognetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.log189FBB78FE1F23F729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e3cd94ab34d1b5cb38d2d62928042acf4a195aacb875e5b3ce61af7e133f5d6c, Size: 2682}"} +{"time":"2026-03-24T17:01:46.544795606+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=37c1c8b11290535a5dccb65faaf1b5e16cbc96ec65898ab418b2ed23847cabd0"} +{"time":"2026-03-24T17:01:46.544900322+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.json189FBB78FD79ECBE17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=37c1c8b11290535a5dccb65faaf1b5e16cbc96ec65898ab418b2ed23847cabd0, Size: 43}"} +{"time":"2026-03-24T17:01:46.546082064+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a171387400e366cd3f49b5662ca5671bbf8af4bf04a935ca313d9b14507b6cf1"} +{"time":"2026-03-24T17:01:46.546183734+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yamlnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yaml189FBB78FDB1FAB34b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/eb44q2tb/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260324%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260324T090146Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a171387400e366cd3f49b5662ca5671bbf8af4bf04a935ca313d9b14507b6cf1, Size: 7766}"} +{"time":"2026-03-24T17:01:46.546930214+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-03-24T17:01:46.587769696+08:00","level":"INFO","msg":"handler: closed","stream_id":"eb44q2tb"} +{"time":"2026-03-24T17:01:46.587910039+08:00","level":"INFO","msg":"sender: closed","stream_id":"eb44q2tb"} +{"time":"2026-03-24T17:01:46.587937144+08:00","level":"INFO","msg":"stream: closed","id":"eb44q2tb"} diff --git a/wandb/run-20260324_164425-eb44q2tb/logs/debug.log b/wandb/run-20260324_164425-eb44q2tb/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6602fcc1f4a7b9623c492e3696d39fc7665bb76a --- /dev/null +++ b/wandb/run-20260324_164425-eb44q2tb/logs/debug.log @@ -0,0 +1,22 @@ +2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_setup.py:_flush():81] Configure stats pid to 1691523 +2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-03-24 16:44:25,716 INFO MainThread:1691523 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/logs/debug.log +2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260324_164425-eb44q2tb/logs/debug-internal.log +2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:init():844] calling init triggers +2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-03-24 16:44:25,717 INFO MainThread:1691523 [wandb_init.py:init():892] starting backend +2026-03-24 16:44:25,961 INFO MainThread:1691523 [wandb_init.py:init():895] sending inform_init request +2026-03-24 16:44:25,966 INFO MainThread:1691523 [wandb_init.py:init():903] backend started and connected +2026-03-24 16:44:25,967 INFO MainThread:1691523 [wandb_init.py:init():973] updated telemetry +2026-03-24 16:44:25,973 INFO MainThread:1691523 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-03-24 16:44:30,384 INFO MainThread:1691523 [wandb_init.py:init():1042] starting run threads in backend +2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_console_start():2524] atexit reg +2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-03-24 16:44:30,488 INFO MainThread:1691523 [wandb_run.py:_redirect():2465] Redirects installed. +2026-03-24 16:44:30,491 INFO MainThread:1691523 [wandb_init.py:init():1082] run started, returning control to user process +2026-03-24 16:44:30,492 INFO MainThread:1691523 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'eb44q2tb', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 20, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 15, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 64, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': True, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal.yaml'} +2026-03-24 17:01:46,448 INFO wandb-AsyncioManager-main:1691523 [service_client.py:_forward_responses():134] Reached EOF. +2026-03-24 17:01:46,448 INFO wandb-AsyncioManager-main:1691523 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/wandb/run-20260325_141344-o1lsxhwu/files/code/train/rl_lladav.py b/wandb/run-20260325_141344-o1lsxhwu/files/code/train/rl_lladav.py new file mode 100644 index 0000000000000000000000000000000000000000..836e692ee6db8e6001d2c5e8fd8e92a650355b5f --- /dev/null +++ b/wandb/run-20260325_141344-o1lsxhwu/files/code/train/rl_lladav.py @@ -0,0 +1,825 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LLaDA-V model and tokenizer") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wandb/run-20260325_141344-o1lsxhwu/files/config.yaml b/wandb/run-20260325_141344-o1lsxhwu/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c6347ea06c73c4c12988d84977ded571f24a1e4 --- /dev/null +++ b/wandb/run-20260325_141344-o1lsxhwu/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lladav.py + e: + l1zvo1vvzkyq01b0wblads1a5x42yuiv: + args: + - config=configs/rl_lladav_video_reversal.yaml + - experiment.current_epoch=27 + codePath: train/rl_lladav.py + codePathLocal: train/rl_lladav.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.4" + disk: + /: + total: "806578724864" + used: "572884140032" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113813504" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lladav.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-03-25T06:13:44.186076Z" + writerId: l1zvo1vvzkyq01b0wblads1a5x42yuiv + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 13 + - 14 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lladav_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: true +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 27 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2_rl +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lladav_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lladav_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lladav +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: o1lsxhwu diff --git a/wandb/run-20260325_141344-o1lsxhwu/files/diff.patch b/wandb/run-20260325_141344-o1lsxhwu/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260325_141344-o1lsxhwu/files/diff.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260325_141344-o1lsxhwu/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260325_141344-o1lsxhwu/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260325_141344-o1lsxhwu/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260325_141344-o1lsxhwu/files/output.log b/wandb/run-20260325_141344-o1lsxhwu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d816f1787f78578ba95b1d411ff7286f62f568c7 --- /dev/null +++ b/wandb/run-20260325_141344-o1lsxhwu/files/output.log @@ -0,0 +1,31 @@ +03/25/2026 14:13:47 - INFO - root - Saving config to rl_lladav_video_reversal/config.yaml +03/25/2026 14:13:47 - INFO - __main__ - Loading LLaDA-V model and tokenizer +Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal/ckpt/optimized +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.72s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +03/25/2026 14:14:14 - INFO - __main__ - Preprocessing Data +03/25/2026 14:14:14 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 181.57 GB, percent = 9.0% +After initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 194.7 GB, percent = 9.7% +After initializing ZeRO optimizer +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 194.31 GB, percent = 9.6% +03/25/2026 14:15:29 - INFO - __main__ - ***** Running inference ***** +03/25/2026 14:15:29 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +03/25/2026 14:19:13 - INFO - __main__ - ***** Running training ***** +03/25/2026 14:19:13 - INFO - __main__ - Num variants = 560 +03/25/2026 14:19:13 - INFO - __main__ - Num training steps = 36 +03/25/2026 14:19:13 - INFO - __main__ - Instantaneous batch size per device = 1 +03/25/2026 14:19:13 - INFO - __main__ - Total train batch size (parallel * accum) = 16 +03/25/2026 14:19:13 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 100%|██████████| 70/70 [18:59<00:00, 16.28s/it] +03/25/2026 14:38:51 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal/ckpt/optimized diff --git a/wandb/run-20260325_141344-o1lsxhwu/files/requirements.txt b/wandb/run-20260325_141344-o1lsxhwu/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53a0646712bb7ab0c9a25728e3226aa2d67dafd0 --- /dev/null +++ b/wandb/run-20260325_141344-o1lsxhwu/files/requirements.txt @@ -0,0 +1,156 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260325_141344-o1lsxhwu/files/wandb-metadata.json b/wandb/run-20260325_141344-o1lsxhwu/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6bfef1ca11e97166d8bbc5b71fd5f35d70145f42 --- /dev/null +++ b/wandb/run-20260325_141344-o1lsxhwu/files/wandb-metadata.json @@ -0,0 +1,93 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-03-25T06:13:44.186076Z", + "args": [ + "config=configs/rl_lladav_video_reversal.yaml", + "experiment.current_epoch=27" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py", + "codePath": "train/rl_lladav.py", + "codePathLocal": "train/rl_lladav.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "572884140032" + } + }, + "memory": { + "total": "2164113813504" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.4", + "writerId": "l1zvo1vvzkyq01b0wblads1a5x42yuiv" +} \ No newline at end of file diff --git a/wandb/run-20260325_141344-o1lsxhwu/files/wandb-summary.json b/wandb/run-20260325_141344-o1lsxhwu/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..863132d7cf0aefd69a6b0768b6e9dda7e73b5dce --- /dev/null +++ b/wandb/run-20260325_141344-o1lsxhwu/files/wandb-summary.json @@ -0,0 +1 @@ +{"_timestamp":1.774420691877827e+09,"_wandb":{"runtime":1504},"_runtime":1504.468829676,"_step":35,"train/loss":-1.004162073135376,"train/learning_rate":5.868240888334637e-07} \ No newline at end of file diff --git a/wandb/run-20260325_141344-o1lsxhwu/logs/debug-core.log b/wandb/run-20260325_141344-o1lsxhwu/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b30e035dce0d202ead8f0f617d6fa6a013bb2aad --- /dev/null +++ b/wandb/run-20260325_141344-o1lsxhwu/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-03-25T14:13:44.251938439+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp8pcfvdtm/port-3139738.txt","pid":3139738,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-25T14:13:44.252935188+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3139738} +{"time":"2026-03-25T14:13:44.252935305+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3139738-3140405-4267943952/socket","Net":"unix"}} +{"time":"2026-03-25T14:13:44.433120413+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-25T14:13:44.44262472+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"o1lsxhwu","id":"1(@)"} +{"time":"2026-03-25T14:13:47.760716873+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"o1lsxhwu","id":"1(@)"} +{"time":"2026-03-25T14:13:52.960453057+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"k11aar64l975"} +{"time":"2026-03-25T14:38:52.319604236+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"k11aar64l975"} +{"time":"2026-03-25T14:38:52.519484083+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"k11aar64l975"} +{"time":"2026-03-25T14:38:52.521932215+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"o1lsxhwu","id":"1(@)"} +{"time":"2026-03-25T14:38:52.522715833+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"o1lsxhwu","id":"1(@)"} +{"time":"2026-03-25T14:39:04.595815534+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-03-25T14:39:04.595904685+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-03-25T14:39:04.59592411+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-03-25T14:39:04.595944561+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-03-25T14:39:04.596088087+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-03-25T14:39:04.596049096+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3139738-3140405-4267943952/socket","Net":"unix"}} +{"time":"2026-03-25T14:39:04.596107078+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-03-25T14:39:04.596157151+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260325_141344-o1lsxhwu/logs/debug-internal.log b/wandb/run-20260325_141344-o1lsxhwu/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..bd9a4ca7838c7066336a9486f53c5a067874cdcc --- /dev/null +++ b/wandb/run-20260325_141344-o1lsxhwu/logs/debug-internal.log @@ -0,0 +1,30 @@ +{"time":"2026-03-25T14:13:44.442763818+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-25T14:13:47.76019088+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-03-25T14:13:47.76042214+08:00","level":"INFO","msg":"stream: created new stream","id":"o1lsxhwu"} +{"time":"2026-03-25T14:13:47.760515342+08:00","level":"INFO","msg":"handler: started","stream_id":"o1lsxhwu"} +{"time":"2026-03-25T14:13:47.760695658+08:00","level":"INFO","msg":"stream: started","id":"o1lsxhwu"} +{"time":"2026-03-25T14:13:47.760739283+08:00","level":"INFO","msg":"writer: started","stream_id":"o1lsxhwu"} +{"time":"2026-03-25T14:13:47.760750058+08:00","level":"INFO","msg":"sender: started","stream_id":"o1lsxhwu"} +{"time":"2026-03-25T14:13:47.764704511+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-03-25T14:13:52.930347695+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T061352Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cc95edf129e17e600ba2653b4d475f6dbf3dbbff19f83bb75eb2e992ba7bc188"} +{"time":"2026-03-25T14:13:52.930510509+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/diff.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/diff.patch18A000E41C3F0B4A29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260325_141344-o1lsxhwu/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T061352Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cc95edf129e17e600ba2653b4d475f6dbf3dbbff19f83bb75eb2e992ba7bc188, Size: 12686}"} +{"time":"2026-03-25T14:13:52.9312635+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T061352Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=907685fb564ed6cb9c3231d3155027944446279e6e9fbd332b363f286395a246"} +{"time":"2026-03-25T14:13:52.931380509+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18A000E41C56AE5B29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260325_141344-o1lsxhwu/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T061352Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=907685fb564ed6cb9c3231d3155027944446279e6e9fbd332b363f286395a246, Size: 12686}"} +{"time":"2026-03-25T14:13:52.93201778+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T061352Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2cb1ebb53e968c6e2af41fa4a74702cf675b7b7f3bf4b03064097a4fd58b9484"} +{"time":"2026-03-25T14:13:52.932141764+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/requirements.txt18A000E41CE6F0914b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260325_141344-o1lsxhwu/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T061352Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2cb1ebb53e968c6e2af41fa4a74702cf675b7b7f3bf4b03064097a4fd58b9484, Size: 2966}"} +{"time":"2026-03-25T14:13:52.934474469+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T061352Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6977ac15db97d35860474d25cd075313ab05ce1ee140f803f1aa6e99697dfb52"} +{"time":"2026-03-25T14:13:52.934676946+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/wandb-metadata.json18A000E41C78EE7529678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260325_141344-o1lsxhwu/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T061352Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6977ac15db97d35860474d25cd075313ab05ce1ee140f803f1aa6e99697dfb52, Size: 2636}"} +{"time":"2026-03-25T14:13:52.941960624+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T061352Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=813137a7872614d1fa57077408db34ddd12071ef89bf5796a57c7696d2385251"} +{"time":"2026-03-25T14:13:52.94206484+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/code/train/rl_lladav.pynetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/code/train/rl_lladav.py18A000E41CD434BA17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260325_141344-o1lsxhwu/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T061352Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=813137a7872614d1fa57077408db34ddd12071ef89bf5796a57c7696d2385251, Size: 32655}"} +{"time":"2026-03-25T14:38:52.469721254+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T063852Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4bcd683f2f77dc11b9de4379aacd88151241e5eb0ac7b8fb2d4afad22302aa3e"} +{"time":"2026-03-25T14:38:52.469832206+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/output.lognetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/output.log18A002413FC96FA329678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260325_141344-o1lsxhwu/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T063852Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4bcd683f2f77dc11b9de4379aacd88151241e5eb0ac7b8fb2d4afad22302aa3e, Size: 2195}"} +{"time":"2026-03-25T14:38:52.470894343+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T063852Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=66d0d44c42d16ed27f0ce58275866002fb6c87d18c078eef95cde92182d2e780"} +{"time":"2026-03-25T14:38:52.471028893+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/wandb-summary.json18A00241402BA7C617e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260325_141344-o1lsxhwu/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T063852Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=66d0d44c42d16ed27f0ce58275866002fb6c87d18c078eef95cde92182d2e780, Size: 175}"} +{"time":"2026-03-25T14:38:52.473695063+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T063852Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2cdf674ca3d49d0d4571426c053d0fdea298eb6cf122e1935f9ac0a544a60d95"} +{"time":"2026-03-25T14:38:52.473898621+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/config.yamlnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/config.yaml18A0024140843E9F4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260325_141344-o1lsxhwu/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal/o1lsxhwu/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260325%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260325T063852Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2cdf674ca3d49d0d4571426c053d0fdea298eb6cf122e1935f9ac0a544a60d95, Size: 7806}"} +{"time":"2026-03-25T14:38:52.474398083+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-03-25T14:38:52.516341739+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-03-25T14:38:52.5219666+08:00","level":"INFO","msg":"stream: closing","id":"o1lsxhwu"} +{"time":"2026-03-25T14:38:52.522000785+08:00","level":"INFO","msg":"handler: closed","stream_id":"o1lsxhwu"} +{"time":"2026-03-25T14:38:52.522111593+08:00","level":"INFO","msg":"sender: closed","stream_id":"o1lsxhwu"} +{"time":"2026-03-25T14:38:52.522128588+08:00","level":"INFO","msg":"stream: closed","id":"o1lsxhwu"} diff --git a/wandb/run-20260325_141344-o1lsxhwu/logs/debug.log b/wandb/run-20260325_141344-o1lsxhwu/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..94938d3abf768b3879ad98fb7ed78a80c5747067 --- /dev/null +++ b/wandb/run-20260325_141344-o1lsxhwu/logs/debug.log @@ -0,0 +1,25 @@ +2026-03-25 14:13:44,189 INFO MainThread:3139738 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-03-25 14:13:44,189 INFO MainThread:3139738 [wandb_setup.py:_flush():81] Configure stats pid to 3139738 +2026-03-25 14:13:44,189 INFO MainThread:3139738 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-03-25 14:13:44,189 INFO MainThread:3139738 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260325_141344-o1lsxhwu/logs/debug.log +2026-03-25 14:13:44,189 INFO MainThread:3139738 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260325_141344-o1lsxhwu/logs/debug-internal.log +2026-03-25 14:13:44,189 INFO MainThread:3139738 [wandb_init.py:init():844] calling init triggers +2026-03-25 14:13:44,189 INFO MainThread:3139738 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-03-25 14:13:44,189 INFO MainThread:3139738 [wandb_init.py:init():892] starting backend +2026-03-25 14:13:44,433 INFO MainThread:3139738 [wandb_init.py:init():895] sending inform_init request +2026-03-25 14:13:44,438 INFO MainThread:3139738 [wandb_init.py:init():903] backend started and connected +2026-03-25 14:13:44,439 INFO MainThread:3139738 [wandb_init.py:init():973] updated telemetry +2026-03-25 14:13:44,445 INFO MainThread:3139738 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-03-25 14:13:47,850 INFO MainThread:3139738 [wandb_init.py:init():1042] starting run threads in backend +2026-03-25 14:13:47,954 INFO MainThread:3139738 [wandb_run.py:_console_start():2524] atexit reg +2026-03-25 14:13:47,954 INFO MainThread:3139738 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-03-25 14:13:47,954 INFO MainThread:3139738 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-03-25 14:13:47,954 INFO MainThread:3139738 [wandb_run.py:_redirect():2465] Redirects installed. +2026-03-25 14:13:47,957 INFO MainThread:3139738 [wandb_init.py:init():1082] run started, returning control to user process +2026-03-25 14:13:47,958 INFO MainThread:3139738 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'o1lsxhwu', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 27, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': True, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal.yaml'} +2026-03-25 14:38:52,317 INFO MainThread:3139738 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lladav_video_reversal/o1lsxhwu +2026-03-25 14:38:52,318 INFO MainThread:3139738 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-03-25 14:38:52,318 INFO MainThread:3139738 [wandb_run.py:_restore():2472] restore +2026-03-25 14:38:52,319 INFO MainThread:3139738 [wandb_run.py:_restore():2478] restore done +2026-03-25 14:38:52,521 INFO MainThread:3139738 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260327_115308-x9tawmeq/files/code/train/rl_lladav.py b/wandb/run-20260327_115308-x9tawmeq/files/code/train/rl_lladav.py new file mode 100644 index 0000000000000000000000000000000000000000..836e692ee6db8e6001d2c5e8fd8e92a650355b5f --- /dev/null +++ b/wandb/run-20260327_115308-x9tawmeq/files/code/train/rl_lladav.py @@ -0,0 +1,825 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LLaDA-V model and tokenizer") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wandb/run-20260327_115308-x9tawmeq/files/config.yaml b/wandb/run-20260327_115308-x9tawmeq/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1c77a9380d8078183b31214cb32424c5fa21cf4 --- /dev/null +++ b/wandb/run-20260327_115308-x9tawmeq/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lladav.py + e: + q6g3oy2oc1xrzf2mk9mquh5x34ywqqgj: + args: + - config=configs/rl_lladav_video_reversal_rl.yaml + - experiment.current_epoch=29 + codePath: train/rl_lladav.py + codePathLocal: train/rl_lladav.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.4" + disk: + /: + total: "806578724864" + used: "573635342336" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113813504" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lladav.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-03-27T03:53:08.600130Z" + writerId: q6g3oy2oc1xrzf2mk9mquh5x34ywqqgj + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 13 + - 14 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lladav_video_reversal_rl.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 29 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2_rl +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lladav_video_reversal_rl/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lladav_video_reversal_rl +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lladav +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/LLaDA-V +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: x9tawmeq diff --git a/wandb/run-20260327_115308-x9tawmeq/files/diff.patch b/wandb/run-20260327_115308-x9tawmeq/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260327_115308-x9tawmeq/files/diff.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260327_115308-x9tawmeq/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260327_115308-x9tawmeq/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260327_115308-x9tawmeq/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260327_115308-x9tawmeq/files/output.log b/wandb/run-20260327_115308-x9tawmeq/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f4888afb38148aad2e30300b103de6fc865cbc40 --- /dev/null +++ b/wandb/run-20260327_115308-x9tawmeq/files/output.log @@ -0,0 +1,31 @@ +03/27/2026 11:53:14 - INFO - root - Saving config to rl_lladav_video_reversal_rl/config.yaml +03/27/2026 11:53:14 - INFO - __main__ - Loading LLaDA-V model and tokenizer +Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal_rl/ckpt/optimized +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.24s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +03/27/2026 11:53:38 - INFO - __main__ - Preprocessing Data +03/27/2026 11:53:38 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 191.89 GB, percent = 9.5% +After initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 211.68 GB, percent = 10.5% +After initializing ZeRO optimizer +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 212.78 GB, percent = 10.6% +03/27/2026 11:54:53 - INFO - __main__ - ***** Running inference ***** +03/27/2026 11:54:53 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +03/27/2026 11:59:19 - INFO - __main__ - ***** Running training ***** +03/27/2026 11:59:19 - INFO - __main__ - Num variants = 544 +03/27/2026 11:59:19 - INFO - __main__ - Num training steps = 35 +03/27/2026 11:59:19 - INFO - __main__ - Instantaneous batch size per device = 1 +03/27/2026 11:59:19 - INFO - __main__ - Total train batch size (parallel * accum) = 16 +03/27/2026 11:59:19 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 100%|██████████| 68/68 [18:35<00:00, 16.41s/it] +03/27/2026 12:18:37 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal_rl/ckpt/optimized diff --git a/wandb/run-20260327_115308-x9tawmeq/files/requirements.txt b/wandb/run-20260327_115308-x9tawmeq/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53a0646712bb7ab0c9a25728e3226aa2d67dafd0 --- /dev/null +++ b/wandb/run-20260327_115308-x9tawmeq/files/requirements.txt @@ -0,0 +1,156 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260327_115308-x9tawmeq/files/wandb-metadata.json b/wandb/run-20260327_115308-x9tawmeq/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a0948426138f39a0c07fc0ba495b47ca29ce8ee0 --- /dev/null +++ b/wandb/run-20260327_115308-x9tawmeq/files/wandb-metadata.json @@ -0,0 +1,93 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-03-27T03:53:08.600130Z", + "args": [ + "config=configs/rl_lladav_video_reversal_rl.yaml", + "experiment.current_epoch=29" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py", + "codePath": "train/rl_lladav.py", + "codePathLocal": "train/rl_lladav.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "573635342336" + } + }, + "memory": { + "total": "2164113813504" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.4", + "writerId": "q6g3oy2oc1xrzf2mk9mquh5x34ywqqgj" +} \ No newline at end of file diff --git a/wandb/run-20260327_115308-x9tawmeq/files/wandb-summary.json b/wandb/run-20260327_115308-x9tawmeq/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..73da2f1c8a327c47bbc3ae36e8db05abed15efeb --- /dev/null +++ b/wandb/run-20260327_115308-x9tawmeq/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":1524.630444805,"_step":34,"train/loss":0.9851624369621277,"train/learning_rate":5.671166329088273e-07,"_timestamp":1.7745850742075238e+09,"_wandb":{"runtime":1524}} \ No newline at end of file diff --git a/wandb/run-20260327_115308-x9tawmeq/logs/debug-core.log b/wandb/run-20260327_115308-x9tawmeq/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..1999ce8e72f9dff9d432e15961f94513783f9c13 --- /dev/null +++ b/wandb/run-20260327_115308-x9tawmeq/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-03-27T11:53:08.667019333+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpx8ke6ei2/port-1069722.txt","pid":1069722,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-27T11:53:08.668110833+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1069722} +{"time":"2026-03-27T11:53:08.668065172+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1069722-1070225-1898119495/socket","Net":"unix"}} +{"time":"2026-03-27T11:53:08.848211661+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-27T11:53:08.857992556+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"x9tawmeq","id":"1(@)"} +{"time":"2026-03-27T11:53:13.786201084+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"x9tawmeq","id":"1(@)"} +{"time":"2026-03-27T11:53:19.008568666+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"1ztb7cjmaz3s"} +{"time":"2026-03-27T12:18:38.5256585+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"1ztb7cjmaz3s"} +{"time":"2026-03-27T12:18:38.733028908+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"1ztb7cjmaz3s"} +{"time":"2026-03-27T12:18:38.733516916+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"x9tawmeq","id":"1(@)"} +{"time":"2026-03-27T12:18:38.734702711+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"x9tawmeq","id":"1(@)"} +{"time":"2026-03-27T12:19:02.285620345+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-03-27T12:19:02.28567832+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-03-27T12:19:02.285685882+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-03-27T12:19:02.285729347+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-03-27T12:19:02.285759449+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1069722-1070225-1898119495/socket","Net":"unix"}} +{"time":"2026-03-27T12:19:02.285924303+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-03-27T12:19:02.285949767+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-03-27T12:19:02.285967978+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260327_115308-x9tawmeq/logs/debug-internal.log b/wandb/run-20260327_115308-x9tawmeq/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..399a97ae07c95efc1d2c8ae078bafbcf7f1a5560 --- /dev/null +++ b/wandb/run-20260327_115308-x9tawmeq/logs/debug-internal.log @@ -0,0 +1,30 @@ +{"time":"2026-03-27T11:53:08.85898605+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-27T11:53:13.785696534+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-03-27T11:53:13.78587201+08:00","level":"INFO","msg":"stream: created new stream","id":"x9tawmeq"} +{"time":"2026-03-27T11:53:13.785967157+08:00","level":"INFO","msg":"handler: started","stream_id":"x9tawmeq"} +{"time":"2026-03-27T11:53:13.786184917+08:00","level":"INFO","msg":"stream: started","id":"x9tawmeq"} +{"time":"2026-03-27T11:53:13.786224369+08:00","level":"INFO","msg":"writer: started","stream_id":"x9tawmeq"} +{"time":"2026-03-27T11:53:13.786223772+08:00","level":"INFO","msg":"sender: started","stream_id":"x9tawmeq"} +{"time":"2026-03-27T11:53:13.788987022+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-03-27T11:53:18.969878721+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T035318Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=eee1ad0ffaf914776f20a7a421042968392b1cab4a7b67978a37667de45fe5a0"} +{"time":"2026-03-27T11:53:18.970085367+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/diff.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/diff.patch18A0966190715D3E17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260327_115308-x9tawmeq/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T035318Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=eee1ad0ffaf914776f20a7a421042968392b1cab4a7b67978a37667de45fe5a0, Size: 12686}"} +{"time":"2026-03-27T11:53:18.97050207+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T035318Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f48e734fedc21b4e470e8072f72822d15edc452c0a53a0e5d376f07db3d10a3b"} +{"time":"2026-03-27T11:53:18.970608805+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/requirements.txt18A0966190207F1629678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260327_115308-x9tawmeq/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T035318Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f48e734fedc21b4e470e8072f72822d15edc452c0a53a0e5d376f07db3d10a3b, Size: 2966}"} +{"time":"2026-03-27T11:53:18.971244492+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T035318Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b0f8d16acfaffcd5f15fb5bdbdffde07a42eddefbed92810195fefc7a5266045"} +{"time":"2026-03-27T11:53:18.971406958+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/wandb-metadata.json18A09661906071304b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260327_115308-x9tawmeq/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T035318Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b0f8d16acfaffcd5f15fb5bdbdffde07a42eddefbed92810195fefc7a5266045, Size: 2639}"} +{"time":"2026-03-27T11:53:18.973041832+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T035318Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3093f76ddc687356777b76f3fc7452b5e43f4a26c7388abf747ffeb371a024d8"} +{"time":"2026-03-27T11:53:18.973121722+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18A09661908E5FE64b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260327_115308-x9tawmeq/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T035318Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3093f76ddc687356777b76f3fc7452b5e43f4a26c7388abf747ffeb371a024d8, Size: 12686}"} +{"time":"2026-03-27T11:53:18.979144981+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T035318Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=931276d40db774c78ec92d81455f8dfd8a4b3d394fad171de4fdf2f2c725b441"} +{"time":"2026-03-27T11:53:18.979229383+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/code/train/rl_lladav.pynetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/code/train/rl_lladav.py18A09661906B928217e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260327_115308-x9tawmeq/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T035318Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=931276d40db774c78ec92d81455f8dfd8a4b3d394fad171de4fdf2f2c725b441, Size: 32655}"} +{"time":"2026-03-27T12:18:38.683023917+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T041838Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ea7d1cbe4626a48550a078d96b388231fd76a222bf89951276355064041e28e7"} +{"time":"2026-03-27T12:18:38.683135577+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/config.yamlnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/config.yaml18A097C3666B78EE17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260327_115308-x9tawmeq/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T041838Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ea7d1cbe4626a48550a078d96b388231fd76a222bf89951276355064041e28e7, Size: 7781}"} +{"time":"2026-03-27T12:18:38.68434742+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T041838Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8213bb8e76251604dbb6cd9d5ffb58768667c882043a4aec539bdf92d0371c6b"} +{"time":"2026-03-27T12:18:38.684463859+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/wandb-summary.json18A097C3665F05EE4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260327_115308-x9tawmeq/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T041838Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8213bb8e76251604dbb6cd9d5ffb58768667c882043a4aec539bdf92d0371c6b, Size: 176}"} +{"time":"2026-03-27T12:18:38.685224518+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T041838Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ab497cf780a286b3a4b372f15cf63e0f6dbe61c40be4cb606ad6b758e6c52b84"} +{"time":"2026-03-27T12:18:38.68532912+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/output.lognetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/output.log18A097C366734FC84b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260327_115308-x9tawmeq/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260327%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260327T041838Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ab497cf780a286b3a4b372f15cf63e0f6dbe61c40be4cb606ad6b758e6c52b84, Size: 2207}"} +{"time":"2026-03-27T12:18:38.686244191+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-03-27T12:18:38.729282918+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-03-27T12:18:38.733548716+08:00","level":"INFO","msg":"stream: closing","id":"x9tawmeq"} +{"time":"2026-03-27T12:18:38.733576046+08:00","level":"INFO","msg":"handler: closed","stream_id":"x9tawmeq"} +{"time":"2026-03-27T12:18:38.733701592+08:00","level":"INFO","msg":"sender: closed","stream_id":"x9tawmeq"} +{"time":"2026-03-27T12:18:38.73371881+08:00","level":"INFO","msg":"stream: closed","id":"x9tawmeq"} diff --git a/wandb/run-20260327_115308-x9tawmeq/logs/debug.log b/wandb/run-20260327_115308-x9tawmeq/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9f2bc51de2a13f90a9dbebe00f8520200e2bc5ba --- /dev/null +++ b/wandb/run-20260327_115308-x9tawmeq/logs/debug.log @@ -0,0 +1,25 @@ +2026-03-27 11:53:08,601 INFO MainThread:1069722 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-03-27 11:53:08,601 INFO MainThread:1069722 [wandb_setup.py:_flush():81] Configure stats pid to 1069722 +2026-03-27 11:53:08,601 INFO MainThread:1069722 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-03-27 11:53:08,601 INFO MainThread:1069722 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260327_115308-x9tawmeq/logs/debug.log +2026-03-27 11:53:08,602 INFO MainThread:1069722 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260327_115308-x9tawmeq/logs/debug-internal.log +2026-03-27 11:53:08,602 INFO MainThread:1069722 [wandb_init.py:init():844] calling init triggers +2026-03-27 11:53:08,602 INFO MainThread:1069722 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-03-27 11:53:08,602 INFO MainThread:1069722 [wandb_init.py:init():892] starting backend +2026-03-27 11:53:08,848 INFO MainThread:1069722 [wandb_init.py:init():895] sending inform_init request +2026-03-27 11:53:08,853 INFO MainThread:1069722 [wandb_init.py:init():903] backend started and connected +2026-03-27 11:53:08,854 INFO MainThread:1069722 [wandb_init.py:init():973] updated telemetry +2026-03-27 11:53:08,860 INFO MainThread:1069722 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-03-27 11:53:13,894 INFO MainThread:1069722 [wandb_init.py:init():1042] starting run threads in backend +2026-03-27 11:53:14,003 INFO MainThread:1069722 [wandb_run.py:_console_start():2524] atexit reg +2026-03-27 11:53:14,003 INFO MainThread:1069722 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-03-27 11:53:14,003 INFO MainThread:1069722 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-03-27 11:53:14,003 INFO MainThread:1069722 [wandb_run.py:_redirect():2465] Redirects installed. +2026-03-27 11:53:14,007 INFO MainThread:1069722 [wandb_init.py:init():1082] run started, returning control to user process +2026-03-27 11:53:14,008 INFO MainThread:1069722 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'x9tawmeq', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal_rl', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 29, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal_rl/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/LLaDA-V', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal_rl.yaml'} +2026-03-27 12:18:38,523 INFO MainThread:1069722 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lladav_video_reversal_rl/x9tawmeq +2026-03-27 12:18:38,524 INFO MainThread:1069722 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-03-27 12:18:38,524 INFO MainThread:1069722 [wandb_run.py:_restore():2472] restore +2026-03-27 12:18:38,524 INFO MainThread:1069722 [wandb_run.py:_restore():2478] restore done +2026-03-27 12:18:38,732 INFO MainThread:1069722 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260404_132744-t5wtr8x3/logs/debug-core.log b/wandb/run-20260404_132744-t5wtr8x3/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..3b7ea451823b1fa140688cc3e7e3a55aca4993d6 --- /dev/null +++ b/wandb/run-20260404_132744-t5wtr8x3/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-04-04T13:27:44.631661977+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpz_try3i9/port-2539439.txt","pid":2539439,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-04T13:27:44.632711754+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2539439} +{"time":"2026-04-04T13:27:44.632680789+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2539439-2539899-3573284903/socket","Net":"unix"}} +{"time":"2026-04-04T13:27:44.81182684+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-04T13:27:44.819373393+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"t5wtr8x3","id":"1(@)"} +{"time":"2026-04-04T13:27:49.546169275+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"t5wtr8x3","id":"1(@)"} +{"time":"2026-04-04T13:27:54.76770114+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"3lq10s2h9mjt"} +{"time":"2026-04-04T13:50:59.153971918+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"3lq10s2h9mjt"} +{"time":"2026-04-04T13:50:59.386532815+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"3lq10s2h9mjt"} +{"time":"2026-04-04T13:50:59.387288457+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"t5wtr8x3","id":"1(@)"} +{"time":"2026-04-04T13:50:59.388373249+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"t5wtr8x3","id":"1(@)"} +{"time":"2026-04-04T13:51:15.780118885+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-04-04T13:51:15.780195814+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-04-04T13:51:15.780212029+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-04-04T13:51:15.780246344+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-04-04T13:51:15.780328694+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2539439-2539899-3573284903/socket","Net":"unix"}} +{"time":"2026-04-04T13:51:15.780419607+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-04-04T13:51:15.780435445+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-04-04T13:51:15.78045846+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260404_132744-t5wtr8x3/logs/debug-internal.log b/wandb/run-20260404_132744-t5wtr8x3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c250fd989fc0dcecbb165a99e84159b1dba52145 --- /dev/null +++ b/wandb/run-20260404_132744-t5wtr8x3/logs/debug-internal.log @@ -0,0 +1,30 @@ +{"time":"2026-04-04T13:27:44.819589809+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-04T13:27:49.545605948+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-04-04T13:27:49.545804307+08:00","level":"INFO","msg":"stream: created new stream","id":"t5wtr8x3"} +{"time":"2026-04-04T13:27:49.545896675+08:00","level":"INFO","msg":"handler: started","stream_id":"t5wtr8x3"} +{"time":"2026-04-04T13:27:49.546153961+08:00","level":"INFO","msg":"stream: started","id":"t5wtr8x3"} +{"time":"2026-04-04T13:27:49.546200547+08:00","level":"INFO","msg":"writer: started","stream_id":"t5wtr8x3"} +{"time":"2026-04-04T13:27:49.546209915+08:00","level":"INFO","msg":"sender: started","stream_id":"t5wtr8x3"} +{"time":"2026-04-04T13:27:49.550620779+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-04-04T13:27:54.733400908+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T052754Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1822c07882f4e6475eeeda0917f594ac4422392ae581d1e7c0ca542b8a7b165d"} +{"time":"2026-04-04T13:27:54.733550535+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18A3102F988CC8D117e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260404_132744-t5wtr8x3/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T052754Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1822c07882f4e6475eeeda0917f594ac4422392ae581d1e7c0ca542b8a7b165d, Size: 12686}"} +{"time":"2026-04-04T13:27:54.734031877+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T052754Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b7f81aec217b3382e1533a116c5b3315244cdd6e6ee402c97ef418fd9642baf1"} +{"time":"2026-04-04T13:27:54.734160708+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/wandb-metadata.json18A3102F9840A80129678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260404_132744-t5wtr8x3/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T052754Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b7f81aec217b3382e1533a116c5b3315244cdd6e6ee402c97ef418fd9642baf1, Size: 2648}"} +{"time":"2026-04-04T13:27:54.735565232+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T052754Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c70463abece5d7c050d54031de2e31b8d6f697d71ace51f1fa1b68e760b1b450"} +{"time":"2026-04-04T13:27:54.735705978+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/requirements.txt18A3102F984E248829678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260404_132744-t5wtr8x3/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T052754Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c70463abece5d7c050d54031de2e31b8d6f697d71ace51f1fa1b68e760b1b450, Size: 2966}"} +{"time":"2026-04-04T13:27:54.737414908+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T052754Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0b1787ab2b1b75d3c3b72fd2c15b9341b3affe019f660985d7047d70d257e412"} +{"time":"2026-04-04T13:27:54.737526506+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/diff.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/diff.patch18A3102F9876736829678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260404_132744-t5wtr8x3/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T052754Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0b1787ab2b1b75d3c3b72fd2c15b9341b3affe019f660985d7047d70d257e412, Size: 12686}"} +{"time":"2026-04-04T13:27:54.742808068+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T052754Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8d4d78b8c4b6464f75712063a147d2ad6f19f97e88fac209020daf635d9b1264"} +{"time":"2026-04-04T13:27:54.742849752+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/code/train/rl_lladav.pynetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/code/train/rl_lladav.py18A3102F9899C8D84b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260404_132744-t5wtr8x3/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T052754Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8d4d78b8c4b6464f75712063a147d2ad6f19f97e88fac209020daf635d9b1264, Size: 32655}"} +{"time":"2026-04-04T13:50:59.313166036+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T055059Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b6241b313ab5bd97d2a214b84ca967261f7bec0c98d50e64ff24335f718e2cc0"} +{"time":"2026-04-04T13:50:59.313305897+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/wandb-summary.json18A31171F78FCE7329678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260404_132744-t5wtr8x3/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T055059Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b6241b313ab5bd97d2a214b84ca967261f7bec0c98d50e64ff24335f718e2cc0, Size: 177}"} +{"time":"2026-04-04T13:50:59.313405367+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T055059Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce105dd7b6edd9f415fed379d9f2f72812f7d95a9ef4da8651adaef42f02d81a"} +{"time":"2026-04-04T13:50:59.313519726+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/config.yamlnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/config.yaml18A31171F7E6552017e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260404_132744-t5wtr8x3/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T055059Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce105dd7b6edd9f415fed379d9f2f72812f7d95a9ef4da8651adaef42f02d81a, Size: 7855}"} +{"time":"2026-04-04T13:50:59.315174898+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T055059Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=29289b2bcb32a3d11788ff9c006b6b502213c1a768358428f059fef1c185c2ff"} +{"time":"2026-04-04T13:50:59.31640984+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/output.lognetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/output.log18A31171F81134CF4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260404_132744-t5wtr8x3/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T055059Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=29289b2bcb32a3d11788ff9c006b6b502213c1a768358428f059fef1c185c2ff, Size: 2235}"} +{"time":"2026-04-04T13:50:59.318482426+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-04-04T13:50:59.382172147+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-04-04T13:50:59.387299599+08:00","level":"INFO","msg":"stream: closing","id":"t5wtr8x3"} +{"time":"2026-04-04T13:50:59.387309154+08:00","level":"INFO","msg":"handler: closed","stream_id":"t5wtr8x3"} +{"time":"2026-04-04T13:50:59.387371784+08:00","level":"INFO","msg":"sender: closed","stream_id":"t5wtr8x3"} +{"time":"2026-04-04T13:50:59.387385465+08:00","level":"INFO","msg":"stream: closed","id":"t5wtr8x3"} diff --git a/wandb/run-20260404_132744-t5wtr8x3/logs/debug.log b/wandb/run-20260404_132744-t5wtr8x3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e01e5a15d8d063136c95c01a088f4f2f2087cdca --- /dev/null +++ b/wandb/run-20260404_132744-t5wtr8x3/logs/debug.log @@ -0,0 +1,25 @@ +2026-04-04 13:27:44,566 INFO MainThread:2539439 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-04 13:27:44,567 INFO MainThread:2539439 [wandb_setup.py:_flush():81] Configure stats pid to 2539439 +2026-04-04 13:27:44,567 INFO MainThread:2539439 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-04 13:27:44,567 INFO MainThread:2539439 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260404_132744-t5wtr8x3/logs/debug.log +2026-04-04 13:27:44,567 INFO MainThread:2539439 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260404_132744-t5wtr8x3/logs/debug-internal.log +2026-04-04 13:27:44,567 INFO MainThread:2539439 [wandb_init.py:init():844] calling init triggers +2026-04-04 13:27:44,567 INFO MainThread:2539439 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-04-04 13:27:44,567 INFO MainThread:2539439 [wandb_init.py:init():892] starting backend +2026-04-04 13:27:44,812 INFO MainThread:2539439 [wandb_init.py:init():895] sending inform_init request +2026-04-04 13:27:44,816 INFO MainThread:2539439 [wandb_init.py:init():903] backend started and connected +2026-04-04 13:27:44,817 INFO MainThread:2539439 [wandb_init.py:init():973] updated telemetry +2026-04-04 13:27:44,823 INFO MainThread:2539439 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-04 13:27:49,654 INFO MainThread:2539439 [wandb_init.py:init():1042] starting run threads in backend +2026-04-04 13:27:49,762 INFO MainThread:2539439 [wandb_run.py:_console_start():2524] atexit reg +2026-04-04 13:27:49,762 INFO MainThread:2539439 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-04 13:27:49,762 INFO MainThread:2539439 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-04 13:27:49,762 INFO MainThread:2539439 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-04 13:27:49,766 INFO MainThread:2539439 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-04 13:27:49,767 INFO MainThread:2539439 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 't5wtr8x3', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal_wo_t_reward', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 11, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal_wo_t_reward/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal_wo_t_reward.yaml'} +2026-04-04 13:50:59,152 INFO MainThread:2539439 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lladav_video_reversal_wo_t_reward/t5wtr8x3 +2026-04-04 13:50:59,153 INFO MainThread:2539439 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-04-04 13:50:59,153 INFO MainThread:2539439 [wandb_run.py:_restore():2472] restore +2026-04-04 13:50:59,153 INFO MainThread:2539439 [wandb_run.py:_restore():2478] restore done +2026-04-04 13:50:59,386 INFO MainThread:2539439 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260404_204122-vtt0lwv8/files/code/train/rl_lladav.py b/wandb/run-20260404_204122-vtt0lwv8/files/code/train/rl_lladav.py new file mode 100644 index 0000000000000000000000000000000000000000..836e692ee6db8e6001d2c5e8fd8e92a650355b5f --- /dev/null +++ b/wandb/run-20260404_204122-vtt0lwv8/files/code/train/rl_lladav.py @@ -0,0 +1,825 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LLaDA-V model and tokenizer") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wandb/run-20260404_204122-vtt0lwv8/files/config.yaml b/wandb/run-20260404_204122-vtt0lwv8/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..677ea28111ee0978343346867dfa64ca7d838103 --- /dev/null +++ b/wandb/run-20260404_204122-vtt0lwv8/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lladav.py + e: + rz7pzi2nzh97ybhyaixehph4td58qxm4: + args: + - config=configs/rl_lladav_video_reversal_wo_t_reward.yaml + - experiment.current_epoch=17 + codePath: train/rl_lladav.py + codePathLocal: train/rl_lladav.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.4" + disk: + /: + total: "806578724864" + used: "620878184448" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113813504" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lladav.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-04-04T12:41:22.555746Z" + writerId: rz7pzi2nzh97ybhyaixehph4td58qxm4 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 13 + - 14 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lladav_video_reversal_wo_t_reward.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 17 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2_rl +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lladav_video_reversal_wo_t_reward/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lladav_video_reversal_wo_t_reward +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lladav +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: vtt0lwv8 diff --git a/wandb/run-20260404_204122-vtt0lwv8/files/diff.patch b/wandb/run-20260404_204122-vtt0lwv8/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260404_204122-vtt0lwv8/files/diff.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260404_204122-vtt0lwv8/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260404_204122-vtt0lwv8/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260404_204122-vtt0lwv8/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260404_204122-vtt0lwv8/files/output.log b/wandb/run-20260404_204122-vtt0lwv8/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2fedcdd45dae424b4d7853bd403c6d0148015b71 --- /dev/null +++ b/wandb/run-20260404_204122-vtt0lwv8/files/output.log @@ -0,0 +1,31 @@ +04/04/2026 20:41:27 - INFO - root - Saving config to rl_lladav_video_reversal_wo_t_reward/config.yaml +04/04/2026 20:41:27 - INFO - __main__ - Loading LLaDA-V model and tokenizer +Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal_wo_t_reward/ckpt/optimized +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:22<00:00, 5.57s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +04/04/2026 20:41:54 - INFO - __main__ - Preprocessing Data +04/04/2026 20:41:54 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 188.98 GB, percent = 9.4% +After initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 206.49 GB, percent = 10.2% +After initializing ZeRO optimizer +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 208.6 GB, percent = 10.3% +04/04/2026 20:43:08 - INFO - __main__ - ***** Running inference ***** +04/04/2026 20:43:08 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +04/04/2026 20:48:06 - INFO - __main__ - ***** Running training ***** +04/04/2026 20:48:06 - INFO - __main__ - Num variants = 480 +04/04/2026 20:48:06 - INFO - __main__ - Num training steps = 31 +04/04/2026 20:48:06 - INFO - __main__ - Instantaneous batch size per device = 1 +04/04/2026 20:48:06 - INFO - __main__ - Total train batch size (parallel * accum) = 16 +04/04/2026 20:48:06 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 100%|██████████| 60/60 [19:19<00:00, 19.32s/it] +04/04/2026 21:08:04 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal_wo_t_reward/ckpt/optimized diff --git a/wandb/run-20260404_204122-vtt0lwv8/files/requirements.txt b/wandb/run-20260404_204122-vtt0lwv8/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53a0646712bb7ab0c9a25728e3226aa2d67dafd0 --- /dev/null +++ b/wandb/run-20260404_204122-vtt0lwv8/files/requirements.txt @@ -0,0 +1,156 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260404_204122-vtt0lwv8/files/wandb-metadata.json b/wandb/run-20260404_204122-vtt0lwv8/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3678d356dc25b4dee61f0c4f740adb3f5ade34ae --- /dev/null +++ b/wandb/run-20260404_204122-vtt0lwv8/files/wandb-metadata.json @@ -0,0 +1,93 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-04-04T12:41:22.555746Z", + "args": [ + "config=configs/rl_lladav_video_reversal_wo_t_reward.yaml", + "experiment.current_epoch=17" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py", + "codePath": "train/rl_lladav.py", + "codePathLocal": "train/rl_lladav.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "620878184448" + } + }, + "memory": { + "total": "2164113813504" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.4", + "writerId": "rz7pzi2nzh97ybhyaixehph4td58qxm4" +} \ No newline at end of file diff --git a/wandb/run-20260404_204122-vtt0lwv8/files/wandb-summary.json b/wandb/run-20260404_204122-vtt0lwv8/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..eb2e6784893e7a2fae4dadd8658d95a0ef37981d --- /dev/null +++ b/wandb/run-20260404_204122-vtt0lwv8/files/wandb-summary.json @@ -0,0 +1 @@ +{"_step":30,"train/loss":-0.9987505078315735,"train/learning_rate":4.7467541558064326e-07,"_wandb":{"runtime":1596},"_runtime":1596.892171667,"_timestamp":1.7753080449594731e+09} \ No newline at end of file diff --git a/wandb/run-20260404_204122-vtt0lwv8/logs/debug-core.log b/wandb/run-20260404_204122-vtt0lwv8/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..1fc847afdb8c40512421069dd1b60e11ce5fb0f8 --- /dev/null +++ b/wandb/run-20260404_204122-vtt0lwv8/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-04-04T20:41:22.622503103+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpn7kfl5qc/port-2919417.txt","pid":2919417,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-04T20:41:22.623068273+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2919417} +{"time":"2026-04-04T20:41:22.623045956+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2919417-2920256-1365586241/socket","Net":"unix"}} +{"time":"2026-04-04T20:41:22.803818877+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-04T20:41:22.810651742+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"vtt0lwv8","id":"1(@)"} +{"time":"2026-04-04T20:41:27.628430208+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"vtt0lwv8","id":"1(@)"} +{"time":"2026-04-04T20:41:32.834501263+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ikosghnpf8xb"} +{"time":"2026-04-04T21:08:04.61558181+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ikosghnpf8xb"} +{"time":"2026-04-04T21:08:04.808990974+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ikosghnpf8xb"} +{"time":"2026-04-04T21:08:04.809831664+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"vtt0lwv8","id":"1(@)"} +{"time":"2026-04-04T21:08:04.810855187+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"vtt0lwv8","id":"1(@)"} +{"time":"2026-04-04T21:08:16.02918399+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-04-04T21:08:16.029237248+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-04-04T21:08:16.029250396+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-04-04T21:08:16.029261087+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-04-04T21:08:16.029328176+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2919417-2920256-1365586241/socket","Net":"unix"}} +{"time":"2026-04-04T21:08:16.029359332+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-04-04T21:08:16.029369495+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-04-04T21:08:16.029381097+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260404_204122-vtt0lwv8/logs/debug.log b/wandb/run-20260404_204122-vtt0lwv8/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bf09ff303f21ea244cd9ec384fdf7558433aea28 --- /dev/null +++ b/wandb/run-20260404_204122-vtt0lwv8/logs/debug.log @@ -0,0 +1,25 @@ +2026-04-04 20:41:22,558 INFO MainThread:2919417 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-04 20:41:22,558 INFO MainThread:2919417 [wandb_setup.py:_flush():81] Configure stats pid to 2919417 +2026-04-04 20:41:22,559 INFO MainThread:2919417 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-04 20:41:22,559 INFO MainThread:2919417 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260404_204122-vtt0lwv8/logs/debug.log +2026-04-04 20:41:22,559 INFO MainThread:2919417 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260404_204122-vtt0lwv8/logs/debug-internal.log +2026-04-04 20:41:22,559 INFO MainThread:2919417 [wandb_init.py:init():844] calling init triggers +2026-04-04 20:41:22,559 INFO MainThread:2919417 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-04-04 20:41:22,559 INFO MainThread:2919417 [wandb_init.py:init():892] starting backend +2026-04-04 20:41:22,804 INFO MainThread:2919417 [wandb_init.py:init():895] sending inform_init request +2026-04-04 20:41:22,807 INFO MainThread:2919417 [wandb_init.py:init():903] backend started and connected +2026-04-04 20:41:22,810 INFO MainThread:2919417 [wandb_init.py:init():973] updated telemetry +2026-04-04 20:41:22,815 INFO MainThread:2919417 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-04 20:41:27,722 INFO MainThread:2919417 [wandb_init.py:init():1042] starting run threads in backend +2026-04-04 20:41:27,829 INFO MainThread:2919417 [wandb_run.py:_console_start():2524] atexit reg +2026-04-04 20:41:27,829 INFO MainThread:2919417 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-04 20:41:27,829 INFO MainThread:2919417 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-04 20:41:27,829 INFO MainThread:2919417 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-04 20:41:27,833 INFO MainThread:2919417 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-04 20:41:27,834 INFO MainThread:2919417 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'vtt0lwv8', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal_wo_t_reward', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 17, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal_wo_t_reward/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal_wo_t_reward.yaml'} +2026-04-04 21:08:04,613 INFO MainThread:2919417 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lladav_video_reversal_wo_t_reward/vtt0lwv8 +2026-04-04 21:08:04,614 INFO MainThread:2919417 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-04-04 21:08:04,614 INFO MainThread:2919417 [wandb_run.py:_restore():2472] restore +2026-04-04 21:08:04,614 INFO MainThread:2919417 [wandb_run.py:_restore():2478] restore done +2026-04-04 21:08:04,809 INFO MainThread:2919417 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260405_002514-x8646h6w/files/code/train/rl_lladav.py b/wandb/run-20260405_002514-x8646h6w/files/code/train/rl_lladav.py new file mode 100644 index 0000000000000000000000000000000000000000..836e692ee6db8e6001d2c5e8fd8e92a650355b5f --- /dev/null +++ b/wandb/run-20260405_002514-x8646h6w/files/code/train/rl_lladav.py @@ -0,0 +1,825 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LLaDA-V model and tokenizer") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wandb/run-20260405_002514-x8646h6w/files/config.yaml b/wandb/run-20260405_002514-x8646h6w/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..541fa66985cfff05fb7eff48fb0bb1e82b98c800 --- /dev/null +++ b/wandb/run-20260405_002514-x8646h6w/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lladav.py + e: + phe5me18gmtea8t6a4zs06bjp9h04qu7: + args: + - config=configs/rl_lladav_video_reversal_wo_t_reward.yaml + - experiment.current_epoch=20 + codePath: train/rl_lladav.py + codePathLocal: train/rl_lladav.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.4" + disk: + /: + total: "806578724864" + used: "620906180608" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113813504" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lladav.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-04-04T16:25:14.974129Z" + writerId: phe5me18gmtea8t6a4zs06bjp9h04qu7 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 13 + - 14 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lladav_video_reversal_wo_t_reward.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 20 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2_rl +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lladav_video_reversal_wo_t_reward/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lladav_video_reversal_wo_t_reward +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lladav +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: x8646h6w diff --git a/wandb/run-20260405_002514-x8646h6w/files/diff.patch b/wandb/run-20260405_002514-x8646h6w/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260405_002514-x8646h6w/files/diff.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260405_002514-x8646h6w/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260405_002514-x8646h6w/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260405_002514-x8646h6w/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260405_002514-x8646h6w/files/output.log b/wandb/run-20260405_002514-x8646h6w/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d76b46baf6a62363a1720e07f56500dfe3b38999 --- /dev/null +++ b/wandb/run-20260405_002514-x8646h6w/files/output.log @@ -0,0 +1,32 @@ +04/05/2026 00:25:19 - INFO - root - Saving config to rl_lladav_video_reversal_wo_t_reward/config.yaml +04/05/2026 00:25:19 - INFO - __main__ - Loading LLaDA-V model and tokenizer +Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal_wo_t_reward/ckpt/optimized +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.33s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +04/05/2026 00:25:44 - INFO - __main__ - Preprocessing Data +04/05/2026 00:25:44 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 196.22 GB, percent = 9.7% +After initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 213.52 GB, percent = 10.6% +After initializing ZeRO optimizer +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 214.47 GB, percent = 10.6% +04/05/2026 00:27:02 - INFO - __main__ - ***** Running inference ***** +04/05/2026 00:27:02 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +04/05/2026 00:31:26 - INFO - __main__ - ***** Running training ***** +04/05/2026 00:31:26 - INFO - __main__ - Num variants = 528 +04/05/2026 00:31:26 - INFO - __main__ - Num training steps = 34 +04/05/2026 00:31:26 - INFO - __main__ - Instantaneous batch size per device = 1 +04/05/2026 00:31:26 - INFO - __main__ - Total train batch size (parallel * accum) = 16 +04/05/2026 00:31:26 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 100%|██████████| 66/66 [18:49<00:00, 17.12s/it] +04/05/2026 00:51:01 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal_wo_t_reward/ckpt/optimized +04/05/2026 00:51:36 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal_wo_t_reward/ckpt/epoch-20 diff --git a/wandb/run-20260405_002514-x8646h6w/files/requirements.txt b/wandb/run-20260405_002514-x8646h6w/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53a0646712bb7ab0c9a25728e3226aa2d67dafd0 --- /dev/null +++ b/wandb/run-20260405_002514-x8646h6w/files/requirements.txt @@ -0,0 +1,156 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260405_002514-x8646h6w/files/wandb-metadata.json b/wandb/run-20260405_002514-x8646h6w/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ccb3afe8e29e10f6e9c78fc72830e30c0d025fdf --- /dev/null +++ b/wandb/run-20260405_002514-x8646h6w/files/wandb-metadata.json @@ -0,0 +1,93 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-04-04T16:25:14.974129Z", + "args": [ + "config=configs/rl_lladav_video_reversal_wo_t_reward.yaml", + "experiment.current_epoch=20" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py", + "codePath": "train/rl_lladav.py", + "codePathLocal": "train/rl_lladav.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "620906180608" + } + }, + "memory": { + "total": "2164113813504" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.4", + "writerId": "phe5me18gmtea8t6a4zs06bjp9h04qu7" +} \ No newline at end of file diff --git a/wandb/run-20260405_002514-x8646h6w/files/wandb-summary.json b/wandb/run-20260405_002514-x8646h6w/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6ec5fd7b48b74879a83b2072380d89738f8102ce --- /dev/null +++ b/wandb/run-20260405_002514-x8646h6w/files/wandb-summary.json @@ -0,0 +1 @@ +{"_timestamp":1.775321414414921e+09,"_runtime":1577.630766289,"_wandb":{"runtime":1577},"_step":33,"train/loss":-0.7742540836334229,"train/learning_rate":5.461341797316508e-07} \ No newline at end of file diff --git a/wandb/run-20260405_002514-x8646h6w/logs/debug-core.log b/wandb/run-20260405_002514-x8646h6w/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..eb970f8604b114a8693a720a901f200f6eb230e1 --- /dev/null +++ b/wandb/run-20260405_002514-x8646h6w/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-04-05T00:25:15.045597432+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpoxssqc6p/port-3104902.txt","pid":3104902,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-05T00:25:15.046324742+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3104902} +{"time":"2026-04-05T00:25:15.04623884+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3104902-3105352-4206189555/socket","Net":"unix"}} +{"time":"2026-04-05T00:25:15.226843842+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-05T00:25:15.235610982+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"x8646h6w","id":"1(@)"} +{"time":"2026-04-05T00:25:19.350136275+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"x8646h6w","id":"1(@)"} +{"time":"2026-04-05T00:25:24.583777594+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"h2hjwjmmdquh"} +{"time":"2026-04-05T00:51:37.104005739+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"h2hjwjmmdquh"} +{"time":"2026-04-05T00:51:37.311508614+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"h2hjwjmmdquh"} +{"time":"2026-04-05T00:51:37.31198324+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"x8646h6w","id":"1(@)"} +{"time":"2026-04-05T00:51:37.312508497+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"x8646h6w","id":"1(@)"} +{"time":"2026-04-05T00:51:58.689206446+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-04-05T00:51:58.689277703+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-04-05T00:51:58.689294088+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-04-05T00:51:58.689325119+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-04-05T00:51:58.689412199+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3104902-3105352-4206189555/socket","Net":"unix"}} +{"time":"2026-04-05T00:51:58.689490993+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-04-05T00:51:58.689507768+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-04-05T00:51:58.689531905+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260405_002514-x8646h6w/logs/debug-internal.log b/wandb/run-20260405_002514-x8646h6w/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..bfd6a44f6f6a0fa54ac1a17e5072b4c776ba3ac8 --- /dev/null +++ b/wandb/run-20260405_002514-x8646h6w/logs/debug-internal.log @@ -0,0 +1,30 @@ +{"time":"2026-04-05T00:25:15.235826335+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-05T00:25:19.347682305+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-04-05T00:25:19.349699589+08:00","level":"INFO","msg":"stream: created new stream","id":"x8646h6w"} +{"time":"2026-04-05T00:25:19.349795299+08:00","level":"INFO","msg":"handler: started","stream_id":"x8646h6w"} +{"time":"2026-04-05T00:25:19.350117245+08:00","level":"INFO","msg":"stream: started","id":"x8646h6w"} +{"time":"2026-04-05T00:25:19.350164195+08:00","level":"INFO","msg":"writer: started","stream_id":"x8646h6w"} +{"time":"2026-04-05T00:25:19.350162756+08:00","level":"INFO","msg":"sender: started","stream_id":"x8646h6w"} +{"time":"2026-04-05T00:25:19.352247024+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-04-05T00:25:24.544045331+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T162524Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4d9fdacb73987ff1f8bc0f953d0dfcf666fe308289012a5311d0590c16e6faa2"} +{"time":"2026-04-05T00:25:24.544174087+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18A33410B83E56024b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_002514-x8646h6w/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T162524Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4d9fdacb73987ff1f8bc0f953d0dfcf666fe308289012a5311d0590c16e6faa2, Size: 12686}"} +{"time":"2026-04-05T00:25:24.545013515+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T162524Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=611df5202b4229df57bf88d407530ab22f8a7e7537c9a372a905d80ba057bdad"} +{"time":"2026-04-05T00:25:24.545139387+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/wandb-metadata.json18A33410B84401BE4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_002514-x8646h6w/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T162524Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=611df5202b4229df57bf88d407530ab22f8a7e7537c9a372a905d80ba057bdad, Size: 2648}"} +{"time":"2026-04-05T00:25:24.546002136+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T162524Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b7e02b228034119110896fb857aae170a343cc4c6b6de4caff469e2fa43e8a60"} +{"time":"2026-04-05T00:25:24.546111638+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/diff.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/diff.patch18A33410B8FCDC6029678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_002514-x8646h6w/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T162524Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b7e02b228034119110896fb857aae170a343cc4c6b6de4caff469e2fa43e8a60, Size: 12686}"} +{"time":"2026-04-05T00:25:24.547339232+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T162524Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=15898305764ee4aa35fc2efc02ed07f0d3a59151d425c73c27c805f89f4af0c9"} +{"time":"2026-04-05T00:25:24.547433371+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/requirements.txt18A33410B9052B5D29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_002514-x8646h6w/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T162524Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=15898305764ee4aa35fc2efc02ed07f0d3a59151d425c73c27c805f89f4af0c9, Size: 2966}"} +{"time":"2026-04-05T00:25:24.559136374+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T162524Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c2c43c5a4e09d1634ca60050b214580d51980c3fdb86f52562b195f656496e5b"} +{"time":"2026-04-05T00:25:24.559227543+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/code/train/rl_lladav.pynetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/code/train/rl_lladav.py18A33410B929954529678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_002514-x8646h6w/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T162524Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c2c43c5a4e09d1634ca60050b214580d51980c3fdb86f52562b195f656496e5b, Size: 32655}"} +{"time":"2026-04-05T00:51:37.260254286+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T165137Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ae903dfc47bb93c4b36655ca28d7514ceb1c9e1329e598b4ced8936c7ae64335"} +{"time":"2026-04-05T00:51:37.260383097+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/config.yamlnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/config.yaml18A3357EE622BBA617e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_002514-x8646h6w/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T165137Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ae903dfc47bb93c4b36655ca28d7514ceb1c9e1329e598b4ced8936c7ae64335, Size: 7855}"} +{"time":"2026-04-05T00:51:37.260953827+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T165137Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=802475ee4e3aa0a4e6d1a92a8eebbf2a969f0b9ef9e745fecae2da9754e3c508"} +{"time":"2026-04-05T00:51:37.261067662+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/wandb-summary.json18A3357EE619D4C029678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_002514-x8646h6w/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T165137Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=802475ee4e3aa0a4e6d1a92a8eebbf2a969f0b9ef9e745fecae2da9754e3c508, Size: 176}"} +{"time":"2026-04-05T00:51:37.262177847+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T165137Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9ca68c99d50982de9b648c4b069769bda788211119a9e58b79e13763e184ae96"} +{"time":"2026-04-05T00:51:37.262345275+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/output.lognetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/output.log18A3357EE626A8D129678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_002514-x8646h6w/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T165137Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9ca68c99d50982de9b648c4b069769bda788211119a9e58b79e13763e184ae96, Size: 2352}"} +{"time":"2026-04-05T00:51:37.2633183+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-04-05T00:51:37.304830748+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-04-05T00:51:37.311995519+08:00","level":"INFO","msg":"stream: closing","id":"x8646h6w"} +{"time":"2026-04-05T00:51:37.312010459+08:00","level":"INFO","msg":"handler: closed","stream_id":"x8646h6w"} +{"time":"2026-04-05T00:51:37.312074916+08:00","level":"INFO","msg":"sender: closed","stream_id":"x8646h6w"} +{"time":"2026-04-05T00:51:37.31208842+08:00","level":"INFO","msg":"stream: closed","id":"x8646h6w"} diff --git a/wandb/run-20260405_002514-x8646h6w/logs/debug.log b/wandb/run-20260405_002514-x8646h6w/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..09e647b2f557f81b628a3c78d43b276517a7f149 --- /dev/null +++ b/wandb/run-20260405_002514-x8646h6w/logs/debug.log @@ -0,0 +1,25 @@ +2026-04-05 00:25:14,976 INFO MainThread:3104902 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-05 00:25:14,976 INFO MainThread:3104902 [wandb_setup.py:_flush():81] Configure stats pid to 3104902 +2026-04-05 00:25:14,976 INFO MainThread:3104902 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-05 00:25:14,976 INFO MainThread:3104902 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260405_002514-x8646h6w/logs/debug.log +2026-04-05 00:25:14,976 INFO MainThread:3104902 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260405_002514-x8646h6w/logs/debug-internal.log +2026-04-05 00:25:14,976 INFO MainThread:3104902 [wandb_init.py:init():844] calling init triggers +2026-04-05 00:25:14,976 INFO MainThread:3104902 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-04-05 00:25:14,976 INFO MainThread:3104902 [wandb_init.py:init():892] starting backend +2026-04-05 00:25:15,227 INFO MainThread:3104902 [wandb_init.py:init():895] sending inform_init request +2026-04-05 00:25:15,231 INFO MainThread:3104902 [wandb_init.py:init():903] backend started and connected +2026-04-05 00:25:15,234 INFO MainThread:3104902 [wandb_init.py:init():973] updated telemetry +2026-04-05 00:25:15,239 INFO MainThread:3104902 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-05 00:25:19,473 INFO MainThread:3104902 [wandb_init.py:init():1042] starting run threads in backend +2026-04-05 00:25:19,578 INFO MainThread:3104902 [wandb_run.py:_console_start():2524] atexit reg +2026-04-05 00:25:19,578 INFO MainThread:3104902 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-05 00:25:19,578 INFO MainThread:3104902 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-05 00:25:19,578 INFO MainThread:3104902 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-05 00:25:19,582 INFO MainThread:3104902 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-05 00:25:19,583 INFO MainThread:3104902 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'x8646h6w', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal_wo_t_reward', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 20, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal_wo_t_reward/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal_wo_t_reward.yaml'} +2026-04-05 00:51:37,102 INFO MainThread:3104902 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lladav_video_reversal_wo_t_reward/x8646h6w +2026-04-05 00:51:37,103 INFO MainThread:3104902 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-04-05 00:51:37,103 INFO MainThread:3104902 [wandb_run.py:_restore():2472] restore +2026-04-05 00:51:37,103 INFO MainThread:3104902 [wandb_run.py:_restore():2478] restore done +2026-04-05 00:51:37,311 INFO MainThread:3104902 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260405_052010-6s2h1xg9/files/code/train/rl_lladav.py b/wandb/run-20260405_052010-6s2h1xg9/files/code/train/rl_lladav.py new file mode 100644 index 0000000000000000000000000000000000000000..836e692ee6db8e6001d2c5e8fd8e92a650355b5f --- /dev/null +++ b/wandb/run-20260405_052010-6s2h1xg9/files/code/train/rl_lladav.py @@ -0,0 +1,825 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LLaDA-V model and tokenizer") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wandb/run-20260405_052010-6s2h1xg9/files/config.yaml b/wandb/run-20260405_052010-6s2h1xg9/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..664fc8df8d6e6d4a3698b7b6e483751fd44ec3e4 --- /dev/null +++ b/wandb/run-20260405_052010-6s2h1xg9/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lladav.py + e: + gc63hsuxgmfnreco0qx3551arglcvntu: + args: + - config=configs/rl_lladav_video_reversal_wo_t_reward.yaml + - experiment.current_epoch=24 + codePath: train/rl_lladav.py + codePathLocal: train/rl_lladav.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.4" + disk: + /: + total: "806578724864" + used: "620931940352" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113813504" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lladav.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-04-04T21:20:10.462893Z" + writerId: gc63hsuxgmfnreco0qx3551arglcvntu + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 13 + - 14 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lladav_video_reversal_wo_t_reward.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 24 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2_rl +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lladav_video_reversal_wo_t_reward/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lladav_video_reversal_wo_t_reward +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lladav +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: 6s2h1xg9 diff --git a/wandb/run-20260405_052010-6s2h1xg9/files/diff.patch b/wandb/run-20260405_052010-6s2h1xg9/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260405_052010-6s2h1xg9/files/diff.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260405_052010-6s2h1xg9/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260405_052010-6s2h1xg9/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260405_052010-6s2h1xg9/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260405_052010-6s2h1xg9/files/output.log b/wandb/run-20260405_052010-6s2h1xg9/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8572c7c58f7c4cd19cc7cf2e82ca548901c1ed27 --- /dev/null +++ b/wandb/run-20260405_052010-6s2h1xg9/files/output.log @@ -0,0 +1,31 @@ +04/05/2026 05:20:15 - INFO - root - Saving config to rl_lladav_video_reversal_wo_t_reward/config.yaml +04/05/2026 05:20:15 - INFO - __main__ - Loading LLaDA-V model and tokenizer +Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal_wo_t_reward/ckpt/optimized +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.35s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +04/05/2026 05:20:39 - INFO - __main__ - Preprocessing Data +04/05/2026 05:20:39 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 187.73 GB, percent = 9.3% +After initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 216.41 GB, percent = 10.7% +After initializing ZeRO optimizer +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 217.62 GB, percent = 10.8% +04/05/2026 05:21:58 - INFO - __main__ - ***** Running inference ***** +04/05/2026 05:21:58 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +04/05/2026 05:26:09 - INFO - __main__ - ***** Running training ***** +04/05/2026 05:26:09 - INFO - __main__ - Num variants = 544 +04/05/2026 05:26:09 - INFO - __main__ - Num training steps = 35 +04/05/2026 05:26:09 - INFO - __main__ - Instantaneous batch size per device = 1 +04/05/2026 05:26:09 - INFO - __main__ - Total train batch size (parallel * accum) = 16 +04/05/2026 05:26:09 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 100%|██████████| 68/68 [19:25<00:00, 17.14s/it] +04/05/2026 05:46:21 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal_wo_t_reward/ckpt/optimized diff --git a/wandb/run-20260405_052010-6s2h1xg9/files/requirements.txt b/wandb/run-20260405_052010-6s2h1xg9/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53a0646712bb7ab0c9a25728e3226aa2d67dafd0 --- /dev/null +++ b/wandb/run-20260405_052010-6s2h1xg9/files/requirements.txt @@ -0,0 +1,156 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260405_052010-6s2h1xg9/files/wandb-metadata.json b/wandb/run-20260405_052010-6s2h1xg9/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..46c90218de92ecdad25b5df5d79ea00d1c79ed77 --- /dev/null +++ b/wandb/run-20260405_052010-6s2h1xg9/files/wandb-metadata.json @@ -0,0 +1,93 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-04-04T21:20:10.462893Z", + "args": [ + "config=configs/rl_lladav_video_reversal_wo_t_reward.yaml", + "experiment.current_epoch=24" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py", + "codePath": "train/rl_lladav.py", + "codePathLocal": "train/rl_lladav.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "620931940352" + } + }, + "memory": { + "total": "2164113813504" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.4", + "writerId": "gc63hsuxgmfnreco0qx3551arglcvntu" +} \ No newline at end of file diff --git a/wandb/run-20260405_052010-6s2h1xg9/files/wandb-summary.json b/wandb/run-20260405_052010-6s2h1xg9/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fb613fa7689115406f9e39d3e42f1595a071b3fc --- /dev/null +++ b/wandb/run-20260405_052010-6s2h1xg9/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":1567},"_runtime":1567.353847384,"train/loss":-0.37550508975982666,"train/learning_rate":5.671166329088273e-07,"_timestamp":1.775339133048693e+09,"_step":34} \ No newline at end of file diff --git a/wandb/run-20260405_052010-6s2h1xg9/logs/debug-core.log b/wandb/run-20260405_052010-6s2h1xg9/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..e52eaf294ceaa5f86190b0acefc60dad4de2f61d --- /dev/null +++ b/wandb/run-20260405_052010-6s2h1xg9/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-04-05T05:20:10.527771232+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpe3dcswnq/port-3360988.txt","pid":3360988,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-05T05:20:10.528825378+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3360988} +{"time":"2026-04-05T05:20:10.528808265+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3360988-3361440-279745179/socket","Net":"unix"}} +{"time":"2026-04-05T05:20:10.709430838+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-05T05:20:10.717010456+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"6s2h1xg9","id":"1(@)"} +{"time":"2026-04-05T05:20:15.038192688+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"6s2h1xg9","id":"1(@)"} +{"time":"2026-04-05T05:20:20.254690976+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"xokmyxbzsrde"} +{"time":"2026-04-05T05:46:22.498304101+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"xokmyxbzsrde"} +{"time":"2026-04-05T05:46:22.727121139+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"xokmyxbzsrde"} +{"time":"2026-04-05T05:46:22.727967569+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"6s2h1xg9","id":"1(@)"} +{"time":"2026-04-05T05:46:22.728784998+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"6s2h1xg9","id":"1(@)"} +{"time":"2026-04-05T05:46:48.105749192+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-04-05T05:46:48.105860958+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-04-05T05:46:48.10587878+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-04-05T05:46:48.105902643+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-04-05T05:46:48.106037554+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3360988-3361440-279745179/socket","Net":"unix"}} +{"time":"2026-04-05T05:46:48.106101756+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-04-05T05:46:48.106125001+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-04-05T05:46:48.106148546+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260405_052010-6s2h1xg9/logs/debug-internal.log b/wandb/run-20260405_052010-6s2h1xg9/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b3b63510999d8d515aa53a6a63eb47bacbab97b9 --- /dev/null +++ b/wandb/run-20260405_052010-6s2h1xg9/logs/debug-internal.log @@ -0,0 +1,30 @@ +{"time":"2026-04-05T05:20:10.717188644+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-05T05:20:15.037570629+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-04-05T05:20:15.037804889+08:00","level":"INFO","msg":"stream: created new stream","id":"6s2h1xg9"} +{"time":"2026-04-05T05:20:15.037886125+08:00","level":"INFO","msg":"handler: started","stream_id":"6s2h1xg9"} +{"time":"2026-04-05T05:20:15.038173863+08:00","level":"INFO","msg":"stream: started","id":"6s2h1xg9"} +{"time":"2026-04-05T05:20:15.038209942+08:00","level":"INFO","msg":"sender: started","stream_id":"6s2h1xg9"} +{"time":"2026-04-05T05:20:15.038211722+08:00","level":"INFO","msg":"writer: started","stream_id":"6s2h1xg9"} +{"time":"2026-04-05T05:20:15.043077399+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-04-05T05:20:20.221720048+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T212020Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c16c24e3a4b206932a1c9cebb05ea157e633eb158b1e21f4fca10eb9dcdb4fcc"} +{"time":"2026-04-05T05:20:20.221881103+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/wandb-metadata.json18A34428D161D3A74b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_052010-6s2h1xg9/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T212020Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c16c24e3a4b206932a1c9cebb05ea157e633eb158b1e21f4fca10eb9dcdb4fcc, Size: 2648}"} +{"time":"2026-04-05T05:20:20.221877929+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T212020Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8b1f4f566e6f5c2f6a05e702c8cf94bd3cc498ea856a6059dd984ee475e06e75"} +{"time":"2026-04-05T05:20:20.222004269+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18A34428D15D2E144b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_052010-6s2h1xg9/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T212020Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8b1f4f566e6f5c2f6a05e702c8cf94bd3cc498ea856a6059dd984ee475e06e75, Size: 12686}"} +{"time":"2026-04-05T05:20:20.222086174+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T212020Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3a3b9fe4aa621e6309870494236838514200f38f141895112b23fc171574b3ae"} +{"time":"2026-04-05T05:20:20.222215667+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/diff.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/diff.patch18A34428D16CC8C64b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_052010-6s2h1xg9/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T212020Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3a3b9fe4aa621e6309870494236838514200f38f141895112b23fc171574b3ae, Size: 12686}"} +{"time":"2026-04-05T05:20:20.223682361+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T212020Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3007d6cdc41c40105e0fdf1d8fdd84a3c4c6bf43f3dddbe97d3a6afb7c3ecc64"} +{"time":"2026-04-05T05:20:20.223834862+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/requirements.txt18A34428D1889EBF4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_052010-6s2h1xg9/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T212020Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3007d6cdc41c40105e0fdf1d8fdd84a3c4c6bf43f3dddbe97d3a6afb7c3ecc64, Size: 2966}"} +{"time":"2026-04-05T05:20:20.232757228+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T212020Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2197dd7eec18e3e3ba0de3c7a7ec2da51ddd6d1d83f78bf6da848423272700c1"} +{"time":"2026-04-05T05:20:20.232819355+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/code/train/rl_lladav.pynetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/code/train/rl_lladav.py18A34428D16519EA29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_052010-6s2h1xg9/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T212020Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2197dd7eec18e3e3ba0de3c7a7ec2da51ddd6d1d83f78bf6da848423272700c1, Size: 32655}"} +{"time":"2026-04-05T05:46:22.654596246+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T214622Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e2e4aa59dfd020fdb6ab6f94db3e1e9a7e3b6c94a94c0832859bc40ec3415669"} +{"time":"2026-04-05T05:46:22.654714455+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/wandb-summary.json18A345949984CEDF17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_052010-6s2h1xg9/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T214622Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e2e4aa59dfd020fdb6ab6f94db3e1e9a7e3b6c94a94c0832859bc40ec3415669, Size: 177}"} +{"time":"2026-04-05T05:46:22.655776181+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T214622Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=13973d536b1d55240b948ef40067d5edc5fe176d212575f80855289fac6a2b72"} +{"time":"2026-04-05T05:46:22.655885647+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/output.lognetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/output.log18A3459499B1B4F54b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_052010-6s2h1xg9/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T214622Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=13973d536b1d55240b948ef40067d5edc5fe176d212575f80855289fac6a2b72, Size: 2234}"} +{"time":"2026-04-05T05:46:22.658482994+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T214622Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=bc9eb77b37bc17c5cb328affb908a5c1613b29d3e0ae7d39448af73fc9b72086"} +{"time":"2026-04-05T05:46:22.658581078+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/config.yamlnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/config.yaml18A3459499C2872717e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_052010-6s2h1xg9/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260404%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260404T214622Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=bc9eb77b37bc17c5cb328affb908a5c1613b29d3e0ae7d39448af73fc9b72086, Size: 7855}"} +{"time":"2026-04-05T05:46:22.659304147+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-04-05T05:46:22.722838888+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-04-05T05:46:22.727995854+08:00","level":"INFO","msg":"stream: closing","id":"6s2h1xg9"} +{"time":"2026-04-05T05:46:22.728013548+08:00","level":"INFO","msg":"handler: closed","stream_id":"6s2h1xg9"} +{"time":"2026-04-05T05:46:22.728110838+08:00","level":"INFO","msg":"sender: closed","stream_id":"6s2h1xg9"} +{"time":"2026-04-05T05:46:22.728120342+08:00","level":"INFO","msg":"stream: closed","id":"6s2h1xg9"} diff --git a/wandb/run-20260405_052010-6s2h1xg9/logs/debug.log b/wandb/run-20260405_052010-6s2h1xg9/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9fffcf2d093c2b392a407e1086dc5dfcf9bc98da --- /dev/null +++ b/wandb/run-20260405_052010-6s2h1xg9/logs/debug.log @@ -0,0 +1,25 @@ +2026-04-05 05:20:10,464 INFO MainThread:3360988 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-05 05:20:10,464 INFO MainThread:3360988 [wandb_setup.py:_flush():81] Configure stats pid to 3360988 +2026-04-05 05:20:10,464 INFO MainThread:3360988 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-05 05:20:10,464 INFO MainThread:3360988 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260405_052010-6s2h1xg9/logs/debug.log +2026-04-05 05:20:10,464 INFO MainThread:3360988 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260405_052010-6s2h1xg9/logs/debug-internal.log +2026-04-05 05:20:10,464 INFO MainThread:3360988 [wandb_init.py:init():844] calling init triggers +2026-04-05 05:20:10,464 INFO MainThread:3360988 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-04-05 05:20:10,465 INFO MainThread:3360988 [wandb_init.py:init():892] starting backend +2026-04-05 05:20:10,709 INFO MainThread:3360988 [wandb_init.py:init():895] sending inform_init request +2026-04-05 05:20:10,714 INFO MainThread:3360988 [wandb_init.py:init():903] backend started and connected +2026-04-05 05:20:10,715 INFO MainThread:3360988 [wandb_init.py:init():973] updated telemetry +2026-04-05 05:20:10,720 INFO MainThread:3360988 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-05 05:20:15,144 INFO MainThread:3360988 [wandb_init.py:init():1042] starting run threads in backend +2026-04-05 05:20:15,251 INFO MainThread:3360988 [wandb_run.py:_console_start():2524] atexit reg +2026-04-05 05:20:15,251 INFO MainThread:3360988 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-05 05:20:15,251 INFO MainThread:3360988 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-05 05:20:15,251 INFO MainThread:3360988 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-05 05:20:15,254 INFO MainThread:3360988 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-05 05:20:15,255 INFO MainThread:3360988 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': '6s2h1xg9', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal_wo_t_reward', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 24, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal_wo_t_reward/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal_wo_t_reward.yaml'} +2026-04-05 05:46:22,496 INFO MainThread:3360988 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lladav_video_reversal_wo_t_reward/6s2h1xg9 +2026-04-05 05:46:22,497 INFO MainThread:3360988 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-04-05 05:46:22,497 INFO MainThread:3360988 [wandb_run.py:_restore():2472] restore +2026-04-05 05:46:22,497 INFO MainThread:3360988 [wandb_run.py:_restore():2478] restore done +2026-04-05 05:46:22,726 INFO MainThread:3360988 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260405_090036-qhv8coi7/files/code/train/rl_lladav.py b/wandb/run-20260405_090036-qhv8coi7/files/code/train/rl_lladav.py new file mode 100644 index 0000000000000000000000000000000000000000..836e692ee6db8e6001d2c5e8fd8e92a650355b5f --- /dev/null +++ b/wandb/run-20260405_090036-qhv8coi7/files/code/train/rl_lladav.py @@ -0,0 +1,825 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LLaDA-V model and tokenizer") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wandb/run-20260405_090036-qhv8coi7/files/config.yaml b/wandb/run-20260405_090036-qhv8coi7/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb2c817ba7412dccc3af92eab591a63cab31fca2 --- /dev/null +++ b/wandb/run-20260405_090036-qhv8coi7/files/config.yaml @@ -0,0 +1,237 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lladav.py + e: + 1famq0zvcif492ayw3rr9al3yj0r79sn: + args: + - config=configs/rl_lladav_video_reversal_wo_t_reward.yaml + - experiment.current_epoch=27 + codePath: train/rl_lladav.py + codePathLocal: train/rl_lladav.py + cpu_count: 96 + cpu_count_logical: 192 + disk: + /: + total: "806578724864" + used: "620949843968" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + host: b1301b38c75f + memory: + total: "2164113813504" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lladav.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-04-05T01:00:36.391076Z" + writerId: 1famq0zvcif492ayw3rr9al3yj0r79sn + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 13 + - 14 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lladav_video_reversal_wo_t_reward.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 27 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2_rl +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lladav_video_reversal_wo_t_reward/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lladav_video_reversal_wo_t_reward +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lladav +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: qhv8coi7 diff --git a/wandb/run-20260405_090036-qhv8coi7/files/diff.patch b/wandb/run-20260405_090036-qhv8coi7/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260405_090036-qhv8coi7/files/diff.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260405_090036-qhv8coi7/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260405_090036-qhv8coi7/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260405_090036-qhv8coi7/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260405_090036-qhv8coi7/files/output.log b/wandb/run-20260405_090036-qhv8coi7/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..23ef867530e405fab80a00ca44b07eca9ba358e5 --- /dev/null +++ b/wandb/run-20260405_090036-qhv8coi7/files/output.log @@ -0,0 +1,31 @@ +04/05/2026 09:00:41 - INFO - root - Saving config to rl_lladav_video_reversal_wo_t_reward/config.yaml +04/05/2026 09:00:41 - INFO - __main__ - Loading LLaDA-V model and tokenizer +Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal_wo_t_reward/ckpt/optimized +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:24<00:00, 6.17s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +04/05/2026 09:01:10 - INFO - __main__ - Preprocessing Data +04/05/2026 09:01:10 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 197.05 GB, percent = 9.8% +After initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 217.1 GB, percent = 10.8% +After initializing ZeRO optimizer +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 217.23 GB, percent = 10.8% +04/05/2026 09:02:31 - INFO - __main__ - ***** Running inference ***** +04/05/2026 09:02:31 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +04/05/2026 09:05:40 - INFO - __main__ - ***** Running training ***** +04/05/2026 09:05:40 - INFO - __main__ - Num variants = 480 +04/05/2026 09:05:40 - INFO - __main__ - Num training steps = 31 +04/05/2026 09:05:40 - INFO - __main__ - Instantaneous batch size per device = 1 +04/05/2026 09:05:40 - INFO - __main__ - Total train batch size (parallel * accum) = 16 +04/05/2026 09:05:40 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 100%|██████████| 60/60 [16:54<00:00, 16.90s/it] +04/05/2026 09:23:22 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal_wo_t_reward/ckpt/optimized diff --git a/wandb/run-20260405_090036-qhv8coi7/files/requirements.txt b/wandb/run-20260405_090036-qhv8coi7/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53a0646712bb7ab0c9a25728e3226aa2d67dafd0 --- /dev/null +++ b/wandb/run-20260405_090036-qhv8coi7/files/requirements.txt @@ -0,0 +1,156 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260405_090036-qhv8coi7/files/wandb-metadata.json b/wandb/run-20260405_090036-qhv8coi7/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0bbd4c287be06fbfc9723577d6eb0677ea2aaa61 --- /dev/null +++ b/wandb/run-20260405_090036-qhv8coi7/files/wandb-metadata.json @@ -0,0 +1,32 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-04-05T01:00:36.391076Z", + "args": [ + "config=configs/rl_lladav_video_reversal_wo_t_reward.yaml", + "experiment.current_epoch=27" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py", + "codePath": "train/rl_lladav.py", + "codePathLocal": "train/rl_lladav.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "disk": { + "/": { + "total": "806578724864", + "used": "620949843968" + } + }, + "memory": { + "total": "2164113813504" + }, + "writerId": "1famq0zvcif492ayw3rr9al3yj0r79sn" +} \ No newline at end of file diff --git a/wandb/run-20260405_090036-qhv8coi7/files/wandb-summary.json b/wandb/run-20260405_090036-qhv8coi7/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..15ae30c037214015b8b9f655235a5ad76cc323f7 --- /dev/null +++ b/wandb/run-20260405_090036-qhv8coi7/files/wandb-summary.json @@ -0,0 +1 @@ +{"_step":30,"_wandb":{"runtime":1360},"_runtime":1360.94784009,"train/loss":-0.5756592154502869,"train/learning_rate":4.7467541558064326e-07,"_timestamp":1.775352153329358e+09} \ No newline at end of file diff --git a/wandb/run-20260405_090036-qhv8coi7/logs/debug-core.log b/wandb/run-20260405_090036-qhv8coi7/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..5f1932801d14b7f335fc94854acf6e763a26be51 --- /dev/null +++ b/wandb/run-20260405_090036-qhv8coi7/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-04-05T09:00:36.457061252+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpsk3eeked/port-3535593.txt","pid":3535593,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-05T09:00:36.458136386+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3535593} +{"time":"2026-04-05T09:00:36.458096205+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3535593-3536158-3136672494/socket","Net":"unix"}} +{"time":"2026-04-05T09:00:36.637636529+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-05T09:00:36.644038819+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"qhv8coi7","id":"1(@)"} +{"time":"2026-04-05T09:00:41.763809809+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"qhv8coi7","id":"1(@)"} +{"time":"2026-04-05T09:00:46.983513092+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"c310m8zqj5cy"} +{"time":"2026-04-05T09:23:22.817143649+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"c310m8zqj5cy"} +{"time":"2026-04-05T09:23:23.021152825+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"c310m8zqj5cy"} +{"time":"2026-04-05T09:23:23.023868178+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"qhv8coi7","id":"1(@)"} +{"time":"2026-04-05T09:23:23.024318245+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"qhv8coi7","id":"1(@)"} +{"time":"2026-04-05T09:23:36.544318235+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-04-05T09:23:36.54439517+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-04-05T09:23:36.544413795+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-04-05T09:23:36.544457136+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-04-05T09:23:36.544531264+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3535593-3536158-3136672494/socket","Net":"unix"}} +{"time":"2026-04-05T09:23:36.544649632+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-04-05T09:23:36.544667225+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-04-05T09:23:36.544694957+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260405_090036-qhv8coi7/logs/debug-internal.log b/wandb/run-20260405_090036-qhv8coi7/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1ce8bd9cdecccc62ced790aae3d623f871cef27a --- /dev/null +++ b/wandb/run-20260405_090036-qhv8coi7/logs/debug-internal.log @@ -0,0 +1,31 @@ +{"time":"2026-04-05T09:00:36.644234875+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-05T09:00:41.706781824+08:00","level":"ERROR","msg":"monitor: failed to initialize GPU resource: monitor: could not get GPU binary port: timeout reading portfile /tmp/wandb-system-monitor-portfile-1169641296"} +{"time":"2026-04-05T09:00:41.763341491+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-04-05T09:00:41.763528298+08:00","level":"INFO","msg":"stream: created new stream","id":"qhv8coi7"} +{"time":"2026-04-05T09:00:41.763604351+08:00","level":"INFO","msg":"handler: started","stream_id":"qhv8coi7"} +{"time":"2026-04-05T09:00:41.763790964+08:00","level":"INFO","msg":"stream: started","id":"qhv8coi7"} +{"time":"2026-04-05T09:00:41.763832648+08:00","level":"INFO","msg":"sender: started","stream_id":"qhv8coi7"} +{"time":"2026-04-05T09:00:41.763841545+08:00","level":"INFO","msg":"writer: started","stream_id":"qhv8coi7"} +{"time":"2026-04-05T09:00:41.765483788+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-04-05T09:00:46.94589057+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T010046Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ee6bc9a5dbb5bed862ce69759c0fe42b68ba1a0461fd33657118ada8958dd211"} +{"time":"2026-04-05T09:00:46.946044702+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18A350306763CBAB4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_090036-qhv8coi7/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T010046Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ee6bc9a5dbb5bed862ce69759c0fe42b68ba1a0461fd33657118ada8958dd211, Size: 12686}"} +{"time":"2026-04-05T09:00:46.946234192+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T010046Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=390fa390c71479a5f44020666b5fb8ff52ae0fb204fad69f3068632327c801fc"} +{"time":"2026-04-05T09:00:46.946367653+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/requirements.txt18A350306753A0D717e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_090036-qhv8coi7/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T010046Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=390fa390c71479a5f44020666b5fb8ff52ae0fb204fad69f3068632327c801fc, Size: 2966}"} +{"time":"2026-04-05T09:00:46.946452003+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T010046Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=dcfcc8fb4b9539c8e6586bbed0d3fbba04af76300a70c10c8fa136795e5025d0"} +{"time":"2026-04-05T09:00:46.946568506+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/wandb-metadata.json18A35030677827694b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_090036-qhv8coi7/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T010046Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=dcfcc8fb4b9539c8e6586bbed0d3fbba04af76300a70c10c8fa136795e5025d0, Size: 971}"} +{"time":"2026-04-05T09:00:46.947345439+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T010046Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5a073986a73d4e289b05c0dd5321f2bdfaf3068aa37033a4462e1489a3b21425"} +{"time":"2026-04-05T09:00:46.947460646+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/diff.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/diff.patch18A350306744FEB329678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_090036-qhv8coi7/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T010046Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5a073986a73d4e289b05c0dd5321f2bdfaf3068aa37033a4462e1489a3b21425, Size: 12686}"} +{"time":"2026-04-05T09:00:46.955932183+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T010046Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d9cdbf955417a16d6146ac156424eec251bf664e9f578fbe949510a71b7a5f31"} +{"time":"2026-04-05T09:00:46.956073591+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/code/train/rl_lladav.pynetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/code/train/rl_lladav.py18A350306745ED1717e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_090036-qhv8coi7/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T010046Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d9cdbf955417a16d6146ac156424eec251bf664e9f578fbe949510a71b7a5f31, Size: 32655}"} +{"time":"2026-04-05T09:23:22.973263809+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T012322Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=bc1ba5da056ed0f97eb6ebcbf0de3f142946dc8c7aa455f6b13d6ffc422be526"} +{"time":"2026-04-05T09:23:22.973383848+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/wandb-summary.json18A3516C20F3CAF54b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_090036-qhv8coi7/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T012322Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=bc1ba5da056ed0f97eb6ebcbf0de3f142946dc8c7aa455f6b13d6ffc422be526, Size: 176}"} +{"time":"2026-04-05T09:23:22.973509201+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T012322Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5ec0e548a8a9dae0c3156c3a5aacc5dad781437e3e9951d4610b03a6954cc618"} +{"time":"2026-04-05T09:23:22.973635802+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/output.lognetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/output.log18A3516C20F4365A4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_090036-qhv8coi7/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T012322Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5ec0e548a8a9dae0c3156c3a5aacc5dad781437e3e9951d4610b03a6954cc618, Size: 2233}"} +{"time":"2026-04-05T09:23:22.977146303+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T012322Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0ba9f7a8d1cbbfb688532570b2934a096ca31927f8a81de4272b6093cfea59c1"} +{"time":"2026-04-05T09:23:22.977313796+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/config.yamlnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/config.yaml18A3516C2105909A17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260405_090036-qhv8coi7/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260405%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260405T012322Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0ba9f7a8d1cbbfb688532570b2934a096ca31927f8a81de4272b6093cfea59c1, Size: 5809}"} +{"time":"2026-04-05T09:23:22.977598422+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-04-05T09:23:23.018757914+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-04-05T09:23:23.023890165+08:00","level":"INFO","msg":"stream: closing","id":"qhv8coi7"} +{"time":"2026-04-05T09:23:23.023905881+08:00","level":"INFO","msg":"handler: closed","stream_id":"qhv8coi7"} +{"time":"2026-04-05T09:23:23.023984109+08:00","level":"INFO","msg":"sender: closed","stream_id":"qhv8coi7"} +{"time":"2026-04-05T09:23:23.023992363+08:00","level":"INFO","msg":"stream: closed","id":"qhv8coi7"} diff --git a/wandb/run-20260405_090036-qhv8coi7/logs/debug.log b/wandb/run-20260405_090036-qhv8coi7/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..599271579541d409e16d2e394959bd6b66624e5b --- /dev/null +++ b/wandb/run-20260405_090036-qhv8coi7/logs/debug.log @@ -0,0 +1,25 @@ +2026-04-05 09:00:36,392 INFO MainThread:3535593 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-05 09:00:36,392 INFO MainThread:3535593 [wandb_setup.py:_flush():81] Configure stats pid to 3535593 +2026-04-05 09:00:36,392 INFO MainThread:3535593 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-05 09:00:36,392 INFO MainThread:3535593 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260405_090036-qhv8coi7/logs/debug.log +2026-04-05 09:00:36,392 INFO MainThread:3535593 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260405_090036-qhv8coi7/logs/debug-internal.log +2026-04-05 09:00:36,392 INFO MainThread:3535593 [wandb_init.py:init():844] calling init triggers +2026-04-05 09:00:36,392 INFO MainThread:3535593 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-04-05 09:00:36,392 INFO MainThread:3535593 [wandb_init.py:init():892] starting backend +2026-04-05 09:00:36,638 INFO MainThread:3535593 [wandb_init.py:init():895] sending inform_init request +2026-04-05 09:00:36,641 INFO MainThread:3535593 [wandb_init.py:init():903] backend started and connected +2026-04-05 09:00:36,642 INFO MainThread:3535593 [wandb_init.py:init():973] updated telemetry +2026-04-05 09:00:36,647 INFO MainThread:3535593 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-05 09:00:41,865 INFO MainThread:3535593 [wandb_init.py:init():1042] starting run threads in backend +2026-04-05 09:00:41,978 INFO MainThread:3535593 [wandb_run.py:_console_start():2524] atexit reg +2026-04-05 09:00:41,978 INFO MainThread:3535593 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-05 09:00:41,978 INFO MainThread:3535593 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-05 09:00:41,978 INFO MainThread:3535593 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-05 09:00:41,982 INFO MainThread:3535593 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-05 09:00:41,983 INFO MainThread:3535593 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'qhv8coi7', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal_wo_t_reward', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 27, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal_wo_t_reward/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal_wo_t_reward.yaml'} +2026-04-05 09:23:22,815 INFO MainThread:3535593 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lladav_video_reversal_wo_t_reward/qhv8coi7 +2026-04-05 09:23:22,816 INFO MainThread:3535593 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-04-05 09:23:22,816 INFO MainThread:3535593 [wandb_run.py:_restore():2472] restore +2026-04-05 09:23:22,816 INFO MainThread:3535593 [wandb_run.py:_restore():2478] restore done +2026-04-05 09:23:23,022 INFO MainThread:3535593 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260406_153231-mh1fe9h6/files/code/train/rl_lladav.py b/wandb/run-20260406_153231-mh1fe9h6/files/code/train/rl_lladav.py new file mode 100644 index 0000000000000000000000000000000000000000..836e692ee6db8e6001d2c5e8fd8e92a650355b5f --- /dev/null +++ b/wandb/run-20260406_153231-mh1fe9h6/files/code/train/rl_lladav.py @@ -0,0 +1,825 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LLaDA-V model and tokenizer") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wandb/run-20260406_153231-mh1fe9h6/files/config.yaml b/wandb/run-20260406_153231-mh1fe9h6/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e62ce899ffc707dc9978fa417492672a617bd630 --- /dev/null +++ b/wandb/run-20260406_153231-mh1fe9h6/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lladav.py + e: + ti8o9gai0s6kkiiv6ghgy9kuvxvxgyrj: + args: + - config=configs/rl_lladav_video_reversal_wo_ub_reward.yaml + - experiment.current_epoch=17 + codePath: train/rl_lladav.py + codePathLocal: train/rl_lladav.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.4" + disk: + /: + total: "806578724864" + used: "620852281344" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113813504" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lladav.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-04-06T07:32:31.706688Z" + writerId: ti8o9gai0s6kkiiv6ghgy9kuvxvxgyrj + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 13 + - 14 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lladav_video_reversal_wo_ub_reward.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 17 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2_rl +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lladav_video_reversal_wo_ub_reward/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lladav_video_reversal_wo_ub_reward +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lladav +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: mh1fe9h6 diff --git a/wandb/run-20260406_153231-mh1fe9h6/files/diff.patch b/wandb/run-20260406_153231-mh1fe9h6/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260406_153231-mh1fe9h6/files/diff.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260406_153231-mh1fe9h6/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260406_153231-mh1fe9h6/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260406_153231-mh1fe9h6/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260406_153231-mh1fe9h6/files/output.log b/wandb/run-20260406_153231-mh1fe9h6/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5d28545ad6fdc208fa22ed3059dcfeecc453873d --- /dev/null +++ b/wandb/run-20260406_153231-mh1fe9h6/files/output.log @@ -0,0 +1,31 @@ +04/06/2026 15:32:36 - INFO - root - Saving config to rl_lladav_video_reversal_wo_ub_reward/config.yaml +04/06/2026 15:32:36 - INFO - __main__ - Loading LLaDA-V model and tokenizer +Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal_wo_ub_reward/ckpt/optimized +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.91s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +04/06/2026 15:33:06 - INFO - __main__ - Preprocessing Data +04/06/2026 15:33:06 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 176.14 GB, percent = 8.7% +After initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 198.61 GB, percent = 9.9% +After initializing ZeRO optimizer +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 199.95 GB, percent = 9.9% +04/06/2026 15:34:26 - INFO - __main__ - ***** Running inference ***** +04/06/2026 15:34:26 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +04/06/2026 15:38:39 - INFO - __main__ - ***** Running training ***** +04/06/2026 15:38:39 - INFO - __main__ - Num variants = 512 +04/06/2026 15:38:39 - INFO - __main__ - Num training steps = 33 +04/06/2026 15:38:39 - INFO - __main__ - Instantaneous batch size per device = 1 +04/06/2026 15:38:39 - INFO - __main__ - Total train batch size (parallel * accum) = 16 +04/06/2026 15:38:39 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 100%|██████████| 64/64 [17:35<00:00, 16.49s/it] +04/06/2026 15:57:04 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal_wo_ub_reward/ckpt/optimized diff --git a/wandb/run-20260406_153231-mh1fe9h6/files/requirements.txt b/wandb/run-20260406_153231-mh1fe9h6/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53a0646712bb7ab0c9a25728e3226aa2d67dafd0 --- /dev/null +++ b/wandb/run-20260406_153231-mh1fe9h6/files/requirements.txt @@ -0,0 +1,156 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260406_153231-mh1fe9h6/files/wandb-metadata.json b/wandb/run-20260406_153231-mh1fe9h6/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1d16a5c90f0a0c60b324a3ee9e2b510ca0ca1593 --- /dev/null +++ b/wandb/run-20260406_153231-mh1fe9h6/files/wandb-metadata.json @@ -0,0 +1,93 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-04-06T07:32:31.706688Z", + "args": [ + "config=configs/rl_lladav_video_reversal_wo_ub_reward.yaml", + "experiment.current_epoch=17" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py", + "codePath": "train/rl_lladav.py", + "codePathLocal": "train/rl_lladav.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "620852281344" + } + }, + "memory": { + "total": "2164113813504" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.4", + "writerId": "ti8o9gai0s6kkiiv6ghgy9kuvxvxgyrj" +} \ No newline at end of file diff --git a/wandb/run-20260406_153231-mh1fe9h6/files/wandb-summary.json b/wandb/run-20260406_153231-mh1fe9h6/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e6bb35e289fb35a7eb13e1a762b371ca2f1ab726 --- /dev/null +++ b/wandb/run-20260406_153231-mh1fe9h6/files/wandb-summary.json @@ -0,0 +1 @@ +{"_step":32,"train/loss":0.5884808301925659,"train/learning_rate":5.237909579118717e-07,"_wandb":{"runtime":1468},"_runtime":1468.493856608,"_timestamp":1.775462173523019e+09} \ No newline at end of file diff --git a/wandb/run-20260406_153231-mh1fe9h6/logs/debug-core.log b/wandb/run-20260406_153231-mh1fe9h6/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..0d32699330164e37d9435cffb7fd5c024dcc0a9a --- /dev/null +++ b/wandb/run-20260406_153231-mh1fe9h6/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-04-06T15:32:31.770713844+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpkyn43uo7/port-511906.txt","pid":511906,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-06T15:32:31.771285531+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":511906} +{"time":"2026-04-06T15:32:31.771273688+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-511906-512219-575198888/socket","Net":"unix"}} +{"time":"2026-04-06T15:32:31.953266966+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-06T15:32:31.961116328+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"mh1fe9h6","id":"1(@)"} +{"time":"2026-04-06T15:32:36.282828086+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"mh1fe9h6","id":"1(@)"} +{"time":"2026-04-06T15:32:41.496212901+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rg335nhslpg2"} +{"time":"2026-04-06T15:57:04.879121928+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rg335nhslpg2"} +{"time":"2026-04-06T15:57:05.0849062+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rg335nhslpg2"} +{"time":"2026-04-06T15:57:05.085257883+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"mh1fe9h6","id":"1(@)"} +{"time":"2026-04-06T15:57:05.085685789+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"mh1fe9h6","id":"1(@)"} +{"time":"2026-04-06T15:57:23.23785121+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-04-06T15:57:23.237940734+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-04-06T15:57:23.23796024+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-04-06T15:57:23.237976345+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-04-06T15:57:23.238088361+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-511906-512219-575198888/socket","Net":"unix"}} +{"time":"2026-04-06T15:57:23.23815476+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-04-06T15:57:23.238181982+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-04-06T15:57:23.238206977+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260406_153231-mh1fe9h6/logs/debug-internal.log b/wandb/run-20260406_153231-mh1fe9h6/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2909807eeb421eb854846e36900be7ea21a932c0 --- /dev/null +++ b/wandb/run-20260406_153231-mh1fe9h6/logs/debug-internal.log @@ -0,0 +1,30 @@ +{"time":"2026-04-06T15:32:31.961271657+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-06T15:32:36.282293521+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-04-06T15:32:36.282482539+08:00","level":"INFO","msg":"stream: created new stream","id":"mh1fe9h6"} +{"time":"2026-04-06T15:32:36.282559978+08:00","level":"INFO","msg":"handler: started","stream_id":"mh1fe9h6"} +{"time":"2026-04-06T15:32:36.282813361+08:00","level":"INFO","msg":"stream: started","id":"mh1fe9h6"} +{"time":"2026-04-06T15:32:36.282861661+08:00","level":"INFO","msg":"sender: started","stream_id":"mh1fe9h6"} +{"time":"2026-04-06T15:32:36.282864781+08:00","level":"INFO","msg":"writer: started","stream_id":"mh1fe9h6"} +{"time":"2026-04-06T15:32:36.285004935+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-04-06T15:32:41.463029327+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T073241Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e5612a86adfc40d53436ba24ec548ada82cb403e5d1b55381d563f81f963ecb6"} +{"time":"2026-04-06T15:32:41.463157138+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/diff.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/diff.patch18A3B427DF5389F529678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260406_153231-mh1fe9h6/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T073241Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e5612a86adfc40d53436ba24ec548ada82cb403e5d1b55381d563f81f963ecb6, Size: 12686}"} +{"time":"2026-04-06T15:32:41.464319857+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T073241Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e7cda02c6cf94964932b297f8ef12206e2b7b54745cdd9b44b795dda56a29299"} +{"time":"2026-04-06T15:32:41.464435193+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18A3B427DF31791017e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260406_153231-mh1fe9h6/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T073241Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e7cda02c6cf94964932b297f8ef12206e2b7b54745cdd9b44b795dda56a29299, Size: 12686}"} +{"time":"2026-04-06T15:32:41.464555484+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T073241Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7b2304fc8815333917cabadbb0306bf0c8923139bebf0448b76f20a1e6e4437b"} +{"time":"2026-04-06T15:32:41.464670045+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/wandb-metadata.json18A3B427DF40D07E17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260406_153231-mh1fe9h6/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T073241Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7b2304fc8815333917cabadbb0306bf0c8923139bebf0448b76f20a1e6e4437b, Size: 2649}"} +{"time":"2026-04-06T15:32:41.464959237+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T073241Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2bbbb3c6447ed4d843427ad56dc278ca7500ac05c4cf283609491c2a579dd099"} +{"time":"2026-04-06T15:32:41.465057586+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/requirements.txt18A3B427DF31FC8B4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260406_153231-mh1fe9h6/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T073241Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2bbbb3c6447ed4d843427ad56dc278ca7500ac05c4cf283609491c2a579dd099, Size: 2966}"} +{"time":"2026-04-06T15:32:41.476109468+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T073241Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cffc4b7cce89c7b74391c68db2ac2a23ff16d1664dedf3e7157126da767c10b5"} +{"time":"2026-04-06T15:32:41.476172524+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/code/train/rl_lladav.pynetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/code/train/rl_lladav.py18A3B427DF7A83E229678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260406_153231-mh1fe9h6/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T073241Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cffc4b7cce89c7b74391c68db2ac2a23ff16d1664dedf3e7157126da767c10b5, Size: 32655}"} +{"time":"2026-04-06T15:57:05.036301207+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T075704Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=34452654e269c190febbe38008cd578bc0903f226e8f4bd39f4e70c500f28da7"} +{"time":"2026-04-06T15:57:05.036374431+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T075704Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8ef35dfa31a12ca7990aa34dac02acb7c31956768d07b8fdf5caa88b3977cc3"} +{"time":"2026-04-06T15:57:05.036428258+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/output.lognetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/output.log18A3B57CA2EAC08A17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260406_153231-mh1fe9h6/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T075704Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=34452654e269c190febbe38008cd578bc0903f226e8f4bd39f4e70c500f28da7, Size: 2235}"} +{"time":"2026-04-06T15:57:05.036487166+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/wandb-summary.json18A3B57CA2C4F6004b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260406_153231-mh1fe9h6/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T075704Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c8ef35dfa31a12ca7990aa34dac02acb7c31956768d07b8fdf5caa88b3977cc3, Size: 175}"} +{"time":"2026-04-06T15:57:05.037739588+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T075704Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2fb9d946cf750d76d3a71a2241d031277a18e2bde89cc3b3c151b4b1df691305"} +{"time":"2026-04-06T15:57:05.037852693+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/config.yamlnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/config.yaml18A3B57CA2DD71254b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260406_153231-mh1fe9h6/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T075704Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2fb9d946cf750d76d3a71a2241d031277a18e2bde89cc3b3c151b4b1df691305, Size: 7859}"} +{"time":"2026-04-06T15:57:05.038433026+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-04-06T15:57:05.078044106+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-04-06T15:57:05.085277635+08:00","level":"INFO","msg":"stream: closing","id":"mh1fe9h6"} +{"time":"2026-04-06T15:57:05.085289839+08:00","level":"INFO","msg":"handler: closed","stream_id":"mh1fe9h6"} +{"time":"2026-04-06T15:57:05.085333112+08:00","level":"INFO","msg":"sender: closed","stream_id":"mh1fe9h6"} +{"time":"2026-04-06T15:57:05.0853447+08:00","level":"INFO","msg":"stream: closed","id":"mh1fe9h6"} diff --git a/wandb/run-20260406_153231-mh1fe9h6/logs/debug.log b/wandb/run-20260406_153231-mh1fe9h6/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..90896ac24d68802e2058e3bb5fce9471ec399089 --- /dev/null +++ b/wandb/run-20260406_153231-mh1fe9h6/logs/debug.log @@ -0,0 +1,25 @@ +2026-04-06 15:32:31,707 INFO MainThread:511906 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-06 15:32:31,707 INFO MainThread:511906 [wandb_setup.py:_flush():81] Configure stats pid to 511906 +2026-04-06 15:32:31,707 INFO MainThread:511906 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-06 15:32:31,708 INFO MainThread:511906 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260406_153231-mh1fe9h6/logs/debug.log +2026-04-06 15:32:31,708 INFO MainThread:511906 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260406_153231-mh1fe9h6/logs/debug-internal.log +2026-04-06 15:32:31,708 INFO MainThread:511906 [wandb_init.py:init():844] calling init triggers +2026-04-06 15:32:31,708 INFO MainThread:511906 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-04-06 15:32:31,708 INFO MainThread:511906 [wandb_init.py:init():892] starting backend +2026-04-06 15:32:31,953 INFO MainThread:511906 [wandb_init.py:init():895] sending inform_init request +2026-04-06 15:32:31,958 INFO MainThread:511906 [wandb_init.py:init():903] backend started and connected +2026-04-06 15:32:31,959 INFO MainThread:511906 [wandb_init.py:init():973] updated telemetry +2026-04-06 15:32:31,965 INFO MainThread:511906 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-06 15:32:36,384 INFO MainThread:511906 [wandb_init.py:init():1042] starting run threads in backend +2026-04-06 15:32:36,488 INFO MainThread:511906 [wandb_run.py:_console_start():2524] atexit reg +2026-04-06 15:32:36,488 INFO MainThread:511906 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-06 15:32:36,488 INFO MainThread:511906 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-06 15:32:36,488 INFO MainThread:511906 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-06 15:32:36,492 INFO MainThread:511906 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-06 15:32:36,493 INFO MainThread:511906 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'mh1fe9h6', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal_wo_ub_reward', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 17, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal_wo_ub_reward/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal_wo_ub_reward.yaml'} +2026-04-06 15:57:04,876 INFO MainThread:511906 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/mh1fe9h6 +2026-04-06 15:57:04,878 INFO MainThread:511906 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-04-06 15:57:04,878 INFO MainThread:511906 [wandb_run.py:_restore():2472] restore +2026-04-06 15:57:04,878 INFO MainThread:511906 [wandb_run.py:_restore():2478] restore done +2026-04-06 15:57:05,084 INFO MainThread:511906 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260407_045326-80gjyuk0/files/code/train/rl_lladav.py b/wandb/run-20260407_045326-80gjyuk0/files/code/train/rl_lladav.py new file mode 100644 index 0000000000000000000000000000000000000000..836e692ee6db8e6001d2c5e8fd8e92a650355b5f --- /dev/null +++ b/wandb/run-20260407_045326-80gjyuk0/files/code/train/rl_lladav.py @@ -0,0 +1,825 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LLaDA-V model and tokenizer") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wandb/run-20260407_045326-80gjyuk0/files/config.yaml b/wandb/run-20260407_045326-80gjyuk0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d65587525790b85ac90031ec8034098a202644d7 --- /dev/null +++ b/wandb/run-20260407_045326-80gjyuk0/files/config.yaml @@ -0,0 +1,237 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lladav.py + e: + ebaktmi1kdzp82u751s6o2s126ekafjt: + args: + - config=configs/rl_lladav_video_reversal_wo_ub_reward.yaml + - experiment.current_epoch=28 + codePath: train/rl_lladav.py + codePathLocal: train/rl_lladav.py + cpu_count: 96 + cpu_count_logical: 192 + disk: + /: + total: "806578724864" + used: "653933711360" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + host: b1301b38c75f + memory: + total: "2164113813504" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lladav.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-04-06T20:53:26.699671Z" + writerId: ebaktmi1kdzp82u751s6o2s126ekafjt + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 13 + - 14 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lladav_video_reversal_wo_ub_reward.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 28 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2_rl +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lladav_video_reversal_wo_ub_reward/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lladav_video_reversal_wo_ub_reward +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lladav +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: 80gjyuk0 diff --git a/wandb/run-20260407_045326-80gjyuk0/files/diff.patch b/wandb/run-20260407_045326-80gjyuk0/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260407_045326-80gjyuk0/files/diff.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260407_045326-80gjyuk0/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260407_045326-80gjyuk0/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260407_045326-80gjyuk0/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260407_045326-80gjyuk0/files/output.log b/wandb/run-20260407_045326-80gjyuk0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7e49efb622d1c5f837f2acaec4f47e11b70a9510 --- /dev/null +++ b/wandb/run-20260407_045326-80gjyuk0/files/output.log @@ -0,0 +1,31 @@ +04/07/2026 04:53:32 - INFO - root - Saving config to rl_lladav_video_reversal_wo_ub_reward/config.yaml +04/07/2026 04:53:32 - INFO - __main__ - Loading LLaDA-V model and tokenizer +Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal_wo_ub_reward/ckpt/optimized +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:24<00:00, 6.00s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +04/07/2026 04:53:59 - INFO - __main__ - Preprocessing Data +04/07/2026 04:53:59 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 168.28 GB, percent = 8.3% +After initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 192.04 GB, percent = 9.5% +After initializing ZeRO optimizer +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 194.54 GB, percent = 9.7% +04/07/2026 04:55:14 - INFO - __main__ - ***** Running inference ***** +04/07/2026 04:55:14 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +04/07/2026 04:58:54 - INFO - __main__ - ***** Running training ***** +04/07/2026 04:58:54 - INFO - __main__ - Num variants = 544 +04/07/2026 04:58:54 - INFO - __main__ - Num training steps = 35 +04/07/2026 04:58:54 - INFO - __main__ - Instantaneous batch size per device = 1 +04/07/2026 04:58:54 - INFO - __main__ - Total train batch size (parallel * accum) = 16 +04/07/2026 04:58:54 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 100%|██████████| 68/68 [17:51<00:00, 15.76s/it] +04/07/2026 05:17:34 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal_wo_ub_reward/ckpt/optimized diff --git a/wandb/run-20260407_045326-80gjyuk0/files/requirements.txt b/wandb/run-20260407_045326-80gjyuk0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53a0646712bb7ab0c9a25728e3226aa2d67dafd0 --- /dev/null +++ b/wandb/run-20260407_045326-80gjyuk0/files/requirements.txt @@ -0,0 +1,156 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260407_045326-80gjyuk0/files/wandb-metadata.json b/wandb/run-20260407_045326-80gjyuk0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..69b3205e69876afdf07ccdd05d2de4aac24b8d44 --- /dev/null +++ b/wandb/run-20260407_045326-80gjyuk0/files/wandb-metadata.json @@ -0,0 +1,32 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-04-06T20:53:26.699671Z", + "args": [ + "config=configs/rl_lladav_video_reversal_wo_ub_reward.yaml", + "experiment.current_epoch=28" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py", + "codePath": "train/rl_lladav.py", + "codePathLocal": "train/rl_lladav.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "disk": { + "/": { + "total": "806578724864", + "used": "653933711360" + } + }, + "memory": { + "total": "2164113813504" + }, + "writerId": "ebaktmi1kdzp82u751s6o2s126ekafjt" +} \ No newline at end of file diff --git a/wandb/run-20260407_045326-80gjyuk0/files/wandb-summary.json b/wandb/run-20260407_045326-80gjyuk0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..277416a6019574653e9b7343221cc41a35331e43 --- /dev/null +++ b/wandb/run-20260407_045326-80gjyuk0/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/loss":0.3736706078052521,"train/learning_rate":5.671166329088273e-07,"_timestamp":1.7755102044776103e+09,"_wandb":{"runtime":1443},"_runtime":1443.322080349,"_step":34} \ No newline at end of file diff --git a/wandb/run-20260407_045326-80gjyuk0/logs/debug-core.log b/wandb/run-20260407_045326-80gjyuk0/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..c8d07853e6483c3d18410ac4d2f872b62e4bb1e6 --- /dev/null +++ b/wandb/run-20260407_045326-80gjyuk0/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-04-07T04:53:26.765529073+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp1pir51ku/port-1268978.txt","pid":1268978,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-07T04:53:26.766610775+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1268978} +{"time":"2026-04-07T04:53:26.76658564+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1268978-1269519-1635121342/socket","Net":"unix"}} +{"time":"2026-04-07T04:53:26.946261734+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-07T04:53:26.954953814+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"80gjyuk0","id":"1(@)"} +{"time":"2026-04-07T04:53:32.19314364+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"80gjyuk0","id":"1(@)"} +{"time":"2026-04-07T04:53:37.403658366+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"x0kbb17h4tbn"} +{"time":"2026-04-07T05:17:35.613447706+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"x0kbb17h4tbn"} +{"time":"2026-04-07T05:17:35.819618035+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"x0kbb17h4tbn"} +{"time":"2026-04-07T05:17:35.821356033+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"80gjyuk0","id":"1(@)"} +{"time":"2026-04-07T05:17:35.821992267+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"80gjyuk0","id":"1(@)"} +{"time":"2026-04-07T05:17:54.809090231+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-04-07T05:17:54.809172143+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-04-07T05:17:54.809188223+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-04-07T05:17:54.809206687+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-04-07T05:17:54.809306853+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1268978-1269519-1635121342/socket","Net":"unix"}} +{"time":"2026-04-07T05:17:54.809375385+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-04-07T05:17:54.809392918+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-04-07T05:17:54.809414303+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260407_045326-80gjyuk0/logs/debug-internal.log b/wandb/run-20260407_045326-80gjyuk0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a23738a91cee22e891b4da0bf1541728d790e44e --- /dev/null +++ b/wandb/run-20260407_045326-80gjyuk0/logs/debug-internal.log @@ -0,0 +1,31 @@ +{"time":"2026-04-07T04:53:26.955083035+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-07T04:53:32.029087854+08:00","level":"ERROR","msg":"monitor: failed to initialize GPU resource: monitor: could not get GPU binary port: timeout reading portfile /tmp/wandb-system-monitor-portfile-64705273"} +{"time":"2026-04-07T04:53:32.190105107+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-04-07T04:53:32.191928181+08:00","level":"INFO","msg":"stream: created new stream","id":"80gjyuk0"} +{"time":"2026-04-07T04:53:32.192199195+08:00","level":"INFO","msg":"handler: started","stream_id":"80gjyuk0"} +{"time":"2026-04-07T04:53:32.193091935+08:00","level":"INFO","msg":"stream: started","id":"80gjyuk0"} +{"time":"2026-04-07T04:53:32.193136517+08:00","level":"INFO","msg":"sender: started","stream_id":"80gjyuk0"} +{"time":"2026-04-07T04:53:32.193145454+08:00","level":"INFO","msg":"writer: started","stream_id":"80gjyuk0"} +{"time":"2026-04-07T04:53:32.194026844+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-04-07T04:53:37.368921923+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T205337Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5135e5e9aaae5765cbca2bd1ba235f11d2c144d41a43ce7cc930de711c44275a"} +{"time":"2026-04-07T04:53:37.369079363+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/requirements.txt18A3DFDCC286D44829678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_045326-80gjyuk0/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T205337Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5135e5e9aaae5765cbca2bd1ba235f11d2c144d41a43ce7cc930de711c44275a, Size: 2966}"} +{"time":"2026-04-07T04:53:37.370392682+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T205337Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a51961dcdbb78c52391339c4e01b0d4bbf986e62303375b84ebe5fdca9e378e4"} +{"time":"2026-04-07T04:53:37.370464356+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T205337Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f4a73eadf1ca92365ca7c8155c7364b4df72a7139f31eda49f641e92c01acd4f"} +{"time":"2026-04-07T04:53:37.370507048+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18A3DFDCC29ECF9029678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_045326-80gjyuk0/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T205337Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a51961dcdbb78c52391339c4e01b0d4bbf986e62303375b84ebe5fdca9e378e4, Size: 12686}"} +{"time":"2026-04-07T04:53:37.370710844+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/diff.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/diff.patch18A3DFDCC2ADCA6217e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_045326-80gjyuk0/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T205337Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f4a73eadf1ca92365ca7c8155c7364b4df72a7139f31eda49f641e92c01acd4f, Size: 12686}"} +{"time":"2026-04-07T04:53:37.372752051+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T205337Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1895c03fc25a7b83fc45822c1f23a5c046f275fb7296a6cac1026a5a48123507"} +{"time":"2026-04-07T04:53:37.372884414+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/wandb-metadata.json18A3DFDCC2A300D54b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_045326-80gjyuk0/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T205337Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1895c03fc25a7b83fc45822c1f23a5c046f275fb7296a6cac1026a5a48123507, Size: 972}"} +{"time":"2026-04-07T04:53:37.382043244+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T205337Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8bbf35383782f4bbe694d4e4f43f325b9a88f1303c6910c4b6ffec6fbc4cfa93"} +{"time":"2026-04-07T04:53:37.382134591+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/code/train/rl_lladav.pynetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/code/train/rl_lladav.py18A3DFDCC2D0E94A17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_045326-80gjyuk0/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T205337Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8bbf35383782f4bbe694d4e4f43f325b9a88f1303c6910c4b6ffec6fbc4cfa93, Size: 32655}"} +{"time":"2026-04-07T05:17:35.770613285+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T211735Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a51b25354f165fa3b0fe5d455dbfa245fe05335a0d758f4e98ab3c7f6fa587c2"} +{"time":"2026-04-07T05:17:35.770705612+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/wandb-summary.json18A3E12BA9E497EB29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_045326-80gjyuk0/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T211735Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a51b25354f165fa3b0fe5d455dbfa245fe05335a0d758f4e98ab3c7f6fa587c2, Size: 176}"} +{"time":"2026-04-07T05:17:35.771460327+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T211735Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d7d57dae33424c1b87a313e2ade31b8f581d6191cb4a9787e27c334dfa0185ab"} +{"time":"2026-04-07T05:17:35.771585757+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/config.yamlnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/config.yaml18A3E12BA9F851F029678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_045326-80gjyuk0/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T211735Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d7d57dae33424c1b87a313e2ade31b8f581d6191cb4a9787e27c334dfa0185ab, Size: 5813}"} +{"time":"2026-04-07T05:17:35.773132883+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T211735Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e7b284c33cc1e3a7a0403de92caa7c575b6325851115092a287e6269e3140643"} +{"time":"2026-04-07T05:17:35.773226571+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/output.lognetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/output.log18A3E12BAA0A2D334b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_045326-80gjyuk0/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T211735Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e7b284c33cc1e3a7a0403de92caa7c575b6325851115092a287e6269e3140643, Size: 2235}"} +{"time":"2026-04-07T05:17:35.773971715+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-04-07T05:17:35.815814802+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-04-07T05:17:35.821387308+08:00","level":"INFO","msg":"stream: closing","id":"80gjyuk0"} +{"time":"2026-04-07T05:17:35.821406273+08:00","level":"INFO","msg":"handler: closed","stream_id":"80gjyuk0"} +{"time":"2026-04-07T05:17:35.821502518+08:00","level":"INFO","msg":"sender: closed","stream_id":"80gjyuk0"} +{"time":"2026-04-07T05:17:35.821512534+08:00","level":"INFO","msg":"stream: closed","id":"80gjyuk0"} diff --git a/wandb/run-20260407_045326-80gjyuk0/logs/debug.log b/wandb/run-20260407_045326-80gjyuk0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1a44c12878658814bdae085bab5f2335fbc6b0ae --- /dev/null +++ b/wandb/run-20260407_045326-80gjyuk0/logs/debug.log @@ -0,0 +1,25 @@ +2026-04-07 04:53:26,702 INFO MainThread:1268978 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-07 04:53:26,702 INFO MainThread:1268978 [wandb_setup.py:_flush():81] Configure stats pid to 1268978 +2026-04-07 04:53:26,702 INFO MainThread:1268978 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-07 04:53:26,702 INFO MainThread:1268978 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260407_045326-80gjyuk0/logs/debug.log +2026-04-07 04:53:26,702 INFO MainThread:1268978 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260407_045326-80gjyuk0/logs/debug-internal.log +2026-04-07 04:53:26,702 INFO MainThread:1268978 [wandb_init.py:init():844] calling init triggers +2026-04-07 04:53:26,702 INFO MainThread:1268978 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-04-07 04:53:26,702 INFO MainThread:1268978 [wandb_init.py:init():892] starting backend +2026-04-07 04:53:26,946 INFO MainThread:1268978 [wandb_init.py:init():895] sending inform_init request +2026-04-07 04:53:26,951 INFO MainThread:1268978 [wandb_init.py:init():903] backend started and connected +2026-04-07 04:53:26,952 INFO MainThread:1268978 [wandb_init.py:init():973] updated telemetry +2026-04-07 04:53:26,958 INFO MainThread:1268978 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-07 04:53:32,292 INFO MainThread:1268978 [wandb_init.py:init():1042] starting run threads in backend +2026-04-07 04:53:32,399 INFO MainThread:1268978 [wandb_run.py:_console_start():2524] atexit reg +2026-04-07 04:53:32,399 INFO MainThread:1268978 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-07 04:53:32,400 INFO MainThread:1268978 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-07 04:53:32,400 INFO MainThread:1268978 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-07 04:53:32,403 INFO MainThread:1268978 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-07 04:53:32,405 INFO MainThread:1268978 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': '80gjyuk0', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal_wo_ub_reward', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 28, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal_wo_ub_reward/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal_wo_ub_reward.yaml'} +2026-04-07 05:17:35,611 INFO MainThread:1268978 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/80gjyuk0 +2026-04-07 05:17:35,612 INFO MainThread:1268978 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-04-07 05:17:35,613 INFO MainThread:1268978 [wandb_run.py:_restore():2472] restore +2026-04-07 05:17:35,613 INFO MainThread:1268978 [wandb_run.py:_restore():2478] restore done +2026-04-07 05:17:35,820 INFO MainThread:1268978 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260407_060237-brhv98wv/files/code/train/rl_lladav.py b/wandb/run-20260407_060237-brhv98wv/files/code/train/rl_lladav.py new file mode 100644 index 0000000000000000000000000000000000000000..836e692ee6db8e6001d2c5e8fd8e92a650355b5f --- /dev/null +++ b/wandb/run-20260407_060237-brhv98wv/files/code/train/rl_lladav.py @@ -0,0 +1,825 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llava_llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_batch, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LLaDA-V model and tokenizer") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, "llava_llada", attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + outputs = model.get_model()(inputs_embeds=noisy_embeds, attention_mask=attn_mask, use_cache=False, return_dict=True) + logits = model.lm_head(outputs.last_hidden_state).float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wandb/run-20260407_060237-brhv98wv/files/config.yaml b/wandb/run-20260407_060237-brhv98wv/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4339cea08e0150f6ceb32f3c412d1db5df99c23 --- /dev/null +++ b/wandb/run-20260407_060237-brhv98wv/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lladav.py + e: + cn8o726roo6eu0nutf9tnaur23zyb4af: + args: + - config=configs/rl_lladav_video_reversal_wo_ub_reward.yaml + - experiment.current_epoch=29 + codePath: train/rl_lladav.py + codePathLocal: train/rl_lladav.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.4" + disk: + /: + total: "806578724864" + used: "653941088256" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113813504" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lladav.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-04-06T22:02:37.045760Z" + writerId: cn8o726roo6eu0nutf9tnaur23zyb4af + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 13 + - 14 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lladav_video_reversal_wo_ub_reward.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 29 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2_rl +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lladav_video_reversal_wo_ub_reward/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lladav_video_reversal_wo_ub_reward +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lladav +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: brhv98wv diff --git a/wandb/run-20260407_060237-brhv98wv/files/diff.patch b/wandb/run-20260407_060237-brhv98wv/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260407_060237-brhv98wv/files/diff.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260407_060237-brhv98wv/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260407_060237-brhv98wv/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a3bf5a0c28b3cb5c6d2118817d173e80747ccb0 --- /dev/null +++ b/wandb/run-20260407_060237-brhv98wv/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,308 @@ +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260407_060237-brhv98wv/files/output.log b/wandb/run-20260407_060237-brhv98wv/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5962df9f079554230b74af91b8fd914d217f2255 --- /dev/null +++ b/wandb/run-20260407_060237-brhv98wv/files/output.log @@ -0,0 +1,31 @@ +04/07/2026 06:02:40 - INFO - root - Saving config to rl_lladav_video_reversal_wo_ub_reward/config.yaml +04/07/2026 06:02:40 - INFO - __main__ - Loading LLaDA-V model and tokenizer +Rank 0: Loaded LLaVA model: ./rl_lladav_video_reversal_wo_ub_reward/ckpt/optimized +Rank 0: Loading vision tower: google/siglip2-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:26<00:00, 6.55s/it] +Rank 0: Model Class: LlavaLLaDAModelLM +04/07/2026 06:03:10 - INFO - __main__ - Preprocessing Data +04/07/2026 06:03:10 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 178.21 GB, percent = 8.8% +After initializing optimizer states +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 206.18 GB, percent = 10.2% +After initializing ZeRO optimizer +MA 16.92 GB Max_MA 16.92 GB CA 18.05 GB Max_CA 18 GB +CPU Virtual Memory: used = 207.48 GB, percent = 10.3% +04/07/2026 06:04:29 - INFO - __main__ - ***** Running inference ***** +04/07/2026 06:04:29 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +04/07/2026 06:08:05 - INFO - __main__ - ***** Running training ***** +04/07/2026 06:08:05 - INFO - __main__ - Num variants = 480 +04/07/2026 06:08:05 - INFO - __main__ - Num training steps = 31 +04/07/2026 06:08:05 - INFO - __main__ - Instantaneous batch size per device = 1 +04/07/2026 06:08:05 - INFO - __main__ - Total train batch size (parallel * accum) = 16 +04/07/2026 06:08:05 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 100%|██████████| 60/60 [17:25<00:00, 17.42s/it] +04/07/2026 06:26:16 - INFO - __main__ - Saved model + tokenizer to rl_lladav_video_reversal_wo_ub_reward/ckpt/optimized diff --git a/wandb/run-20260407_060237-brhv98wv/files/requirements.txt b/wandb/run-20260407_060237-brhv98wv/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53a0646712bb7ab0c9a25728e3226aa2d67dafd0 --- /dev/null +++ b/wandb/run-20260407_060237-brhv98wv/files/requirements.txt @@ -0,0 +1,156 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260407_060237-brhv98wv/files/wandb-metadata.json b/wandb/run-20260407_060237-brhv98wv/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0869bd1bc275364afcc1150c724de324e5e5a0f6 --- /dev/null +++ b/wandb/run-20260407_060237-brhv98wv/files/wandb-metadata.json @@ -0,0 +1,93 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-04-06T22:02:37.045760Z", + "args": [ + "config=configs/rl_lladav_video_reversal_wo_ub_reward.yaml", + "experiment.current_epoch=29" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lladav.py", + "codePath": "train/rl_lladav.py", + "codePathLocal": "train/rl_lladav.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "653941088256" + } + }, + "memory": { + "total": "2164113813504" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.4", + "writerId": "cn8o726roo6eu0nutf9tnaur23zyb4af" +} \ No newline at end of file diff --git a/wandb/run-20260407_060237-brhv98wv/files/wandb-summary.json b/wandb/run-20260407_060237-brhv98wv/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ddf8bdcb246032c65f2b3dc3d8048630854fb04d --- /dev/null +++ b/wandb/run-20260407_060237-brhv98wv/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":1417},"_runtime":1417.139539136,"_step":30,"train/loss":-0.5735836625099182,"train/learning_rate":4.7467541558064326e-07,"_timestamp":1.7755143282632387e+09} \ No newline at end of file diff --git a/wandb/run-20260407_060237-brhv98wv/logs/debug-core.log b/wandb/run-20260407_060237-brhv98wv/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..274c699e09b302ee2569e1178fe253970a2f0712 --- /dev/null +++ b/wandb/run-20260407_060237-brhv98wv/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-04-07T06:02:37.110297593+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp2fyc34fe/port-1346718.txt","pid":1346718,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-07T06:02:37.11083426+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1346718} +{"time":"2026-04-07T06:02:37.110819654+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1346718-1347322-78017562/socket","Net":"unix"}} +{"time":"2026-04-07T06:02:37.291864949+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-07T06:02:37.29925642+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"brhv98wv","id":"1(@)"} +{"time":"2026-04-07T06:02:40.51436947+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"brhv98wv","id":"1(@)"} +{"time":"2026-04-07T06:02:45.734588209+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fqzdv9carrxn"} +{"time":"2026-04-07T06:26:17.760917137+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fqzdv9carrxn"} +{"time":"2026-04-07T06:26:17.961364719+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fqzdv9carrxn"} +{"time":"2026-04-07T06:26:17.967396044+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"brhv98wv","id":"1(@)"} +{"time":"2026-04-07T06:26:17.968000436+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"brhv98wv","id":"1(@)"} +{"time":"2026-04-07T06:26:43.544764961+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-04-07T06:26:43.544829706+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-04-07T06:26:43.544847716+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-04-07T06:26:43.544873355+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-04-07T06:26:43.544971412+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1346718-1347322-78017562/socket","Net":"unix"}} +{"time":"2026-04-07T06:26:43.546712666+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-04-07T06:26:43.546731736+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-04-07T06:26:43.546754356+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260407_060237-brhv98wv/logs/debug-internal.log b/wandb/run-20260407_060237-brhv98wv/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8a1f69ea7cf0217fc292dfa243e6af82ad0fa04f --- /dev/null +++ b/wandb/run-20260407_060237-brhv98wv/logs/debug-internal.log @@ -0,0 +1,30 @@ +{"time":"2026-04-07T06:02:37.299397482+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-07T06:02:40.513856069+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-04-07T06:02:40.514066914+08:00","level":"INFO","msg":"stream: created new stream","id":"brhv98wv"} +{"time":"2026-04-07T06:02:40.514144001+08:00","level":"INFO","msg":"handler: started","stream_id":"brhv98wv"} +{"time":"2026-04-07T06:02:40.51435184+08:00","level":"INFO","msg":"stream: started","id":"brhv98wv"} +{"time":"2026-04-07T06:02:40.514388297+08:00","level":"INFO","msg":"sender: started","stream_id":"brhv98wv"} +{"time":"2026-04-07T06:02:40.514429292+08:00","level":"INFO","msg":"writer: started","stream_id":"brhv98wv"} +{"time":"2026-04-07T06:02:40.519784721+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-04-07T06:02:45.697201763+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T220245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce8c09d5ed2e3ed171ed421b5485d4edd81c6aa77bda8ef69d3fe7a592b6c30e"} +{"time":"2026-04-07T06:02:45.697333251+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/diff.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/diff.patch18A3E3A29E42AA5717e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_060237-brhv98wv/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T220245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce8c09d5ed2e3ed171ed421b5485d4edd81c6aa77bda8ef69d3fe7a592b6c30e, Size: 12686}"} +{"time":"2026-04-07T06:02:45.698997456+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T220245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2b6bb79ddf1c55d90ab57cf2c1b6497cbc03287b9a6f6ba60dcf0e43de62e76e"} +{"time":"2026-04-07T06:02:45.699120892+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/requirements.txt18A3E3A29E39D6CE29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_060237-brhv98wv/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T220245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2b6bb79ddf1c55d90ab57cf2c1b6497cbc03287b9a6f6ba60dcf0e43de62e76e, Size: 2966}"} +{"time":"2026-04-07T06:02:45.699751344+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T220245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ec6a312fd6f8369f8fbe9a39e95056e4fba7e503968c423761e8bbee91fbb1b5"} +{"time":"2026-04-07T06:02:45.699824985+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18A3E3A29E65347D17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_060237-brhv98wv/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T220245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ec6a312fd6f8369f8fbe9a39e95056e4fba7e503968c423761e8bbee91fbb1b5, Size: 12686}"} +{"time":"2026-04-07T06:02:45.700638321+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T220245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=804a22c0f9a23dd76db4dc396e9777b23766d2cd81a267798a3ba91f915a6773"} +{"time":"2026-04-07T06:02:45.700754725+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/wandb-metadata.json18A3E3A29E579A104b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_060237-brhv98wv/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T220245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=804a22c0f9a23dd76db4dc396e9777b23766d2cd81a267798a3ba91f915a6773, Size: 2649}"} +{"time":"2026-04-07T06:02:45.71191033+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T220245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9dfc1ed0d9e83dd0904e738458dec592ef62632e526f4aca4ecabedc195b8a40"} +{"time":"2026-04-07T06:02:45.71199042+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/code/train/rl_lladav.pynetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/code/train/rl_lladav.py18A3E3A29E4DC2434b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_060237-brhv98wv/files/code/train/rl_lladav.py, Name: code/train/rl_lladav.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/code/train/rl_lladav.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T220245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9dfc1ed0d9e83dd0904e738458dec592ef62632e526f4aca4ecabedc195b8a40, Size: 32655}"} +{"time":"2026-04-07T06:26:17.913572812+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T222617Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9d28ad5a5af8991795c637ad3bac0e5a3c82202e0cd74c7552c6c1e63f13522a"} +{"time":"2026-04-07T06:26:17.913666505+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/config.yamlnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/config.yaml18A3E4EB6CC7A5C829678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_060237-brhv98wv/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T222617Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9d28ad5a5af8991795c637ad3bac0e5a3c82202e0cd74c7552c6c1e63f13522a, Size: 7859}"} +{"time":"2026-04-07T06:26:17.913662415+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T222617Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2cbffd32603f4a1f2e7c4ddda8c71d2bd88d68688006f4532890f59fddbc0946"} +{"time":"2026-04-07T06:26:17.91378558+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/wandb-summary.json18A3E4EB6CC5CE7729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_060237-brhv98wv/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T222617Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2cbffd32603f4a1f2e7c4ddda8c71d2bd88d68688006f4532890f59fddbc0946, Size: 178}"} +{"time":"2026-04-07T06:26:17.913841651+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T222617Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c764a5ff18484531ce328c5ca8ecbf9c9c9cf289b5b9cb81c0fb4214083e4426"} +{"time":"2026-04-07T06:26:17.91395013+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/output.lognetdisk/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/output.log18A3E4EB6CBA4BBF4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260407_060237-brhv98wv/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260406%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260406T222617Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c764a5ff18484531ce328c5ca8ecbf9c9c9cf289b5b9cb81c0fb4214083e4426, Size: 2237}"} +{"time":"2026-04-07T06:26:17.914873177+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-04-07T06:26:17.957923056+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-04-07T06:26:17.96742289+08:00","level":"INFO","msg":"stream: closing","id":"brhv98wv"} +{"time":"2026-04-07T06:26:17.967441434+08:00","level":"INFO","msg":"handler: closed","stream_id":"brhv98wv"} +{"time":"2026-04-07T06:26:17.96754279+08:00","level":"INFO","msg":"sender: closed","stream_id":"brhv98wv"} +{"time":"2026-04-07T06:26:17.967556144+08:00","level":"INFO","msg":"stream: closed","id":"brhv98wv"} diff --git a/wandb/run-20260407_060237-brhv98wv/logs/debug.log b/wandb/run-20260407_060237-brhv98wv/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..499826760fa7cb97e4a05d00415945f007d397ce --- /dev/null +++ b/wandb/run-20260407_060237-brhv98wv/logs/debug.log @@ -0,0 +1,25 @@ +2026-04-07 06:02:37,047 INFO MainThread:1346718 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-07 06:02:37,047 INFO MainThread:1346718 [wandb_setup.py:_flush():81] Configure stats pid to 1346718 +2026-04-07 06:02:37,047 INFO MainThread:1346718 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-07 06:02:37,047 INFO MainThread:1346718 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260407_060237-brhv98wv/logs/debug.log +2026-04-07 06:02:37,047 INFO MainThread:1346718 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260407_060237-brhv98wv/logs/debug-internal.log +2026-04-07 06:02:37,047 INFO MainThread:1346718 [wandb_init.py:init():844] calling init triggers +2026-04-07 06:02:37,047 INFO MainThread:1346718 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lladav.py'}} +2026-04-07 06:02:37,047 INFO MainThread:1346718 [wandb_init.py:init():892] starting backend +2026-04-07 06:02:37,292 INFO MainThread:1346718 [wandb_init.py:init():895] sending inform_init request +2026-04-07 06:02:37,296 INFO MainThread:1346718 [wandb_init.py:init():903] backend started and connected +2026-04-07 06:02:37,297 INFO MainThread:1346718 [wandb_init.py:init():973] updated telemetry +2026-04-07 06:02:37,303 INFO MainThread:1346718 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-07 06:02:40,620 INFO MainThread:1346718 [wandb_init.py:init():1042] starting run threads in backend +2026-04-07 06:02:40,728 INFO MainThread:1346718 [wandb_run.py:_console_start():2524] atexit reg +2026-04-07 06:02:40,729 INFO MainThread:1346718 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-07 06:02:40,729 INFO MainThread:1346718 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-07 06:02:40,729 INFO MainThread:1346718 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-07 06:02:40,733 INFO MainThread:1346718 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-07 06:02:40,733 INFO MainThread:1346718 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'brhv98wv', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lladav_video_reversal_wo_ub_reward', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 29, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2_rl', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lladav_video_reversal_wo_ub_reward/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lladav_video_reversal_3_21/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lladav', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lladav_video_reversal_wo_ub_reward.yaml'} +2026-04-07 06:26:17,757 INFO MainThread:1346718 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lladav_video_reversal_wo_ub_reward/brhv98wv +2026-04-07 06:26:17,759 INFO MainThread:1346718 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-04-07 06:26:17,759 INFO MainThread:1346718 [wandb_run.py:_restore():2472] restore +2026-04-07 06:26:17,759 INFO MainThread:1346718 [wandb_run.py:_restore():2478] restore done +2026-04-07 06:26:17,966 INFO MainThread:1346718 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260529_013322-be4lg0e8/files/code/train/sft_lavida_video_reversal.py b/wandb/run-20260529_013322-be4lg0e8/files/code/train/sft_lavida_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..a1833b6f03980216acc96416b3c7c7f749f6bdab --- /dev/null +++ b/wandb/run-20260529_013322-be4lg0e8/files/code/train/sft_lavida_video_reversal.py @@ -0,0 +1,802 @@ +import os +import sys +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LAVIDA_ROOT = os.path.join(REPO_ROOT, "LaViDa") +sys.path.insert(0, LAVIDA_ROOT) +sys.path.insert(1, REPO_ROOT) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator, InitProcessGroupKwargs +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +# LaViDa official / LLaVA +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_pyav, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates +# from accelerate import DistributedDataParallelKwargs #添加 + +logger = get_logger(__name__, log_level="INFO") + +LAVIDA_MODEL_NAME = "llava_llada" + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) # (L,) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + """ + 统一处理 Image 和 Video + 返回 (media_tensor_list, sizes, video_stats) + """ + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + # 使用官方提供的 decord 函数处理视频 + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + + # 处理成 tensor + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + # 提取 size + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + """通用媒体路径解析""" + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class SFTVariantDatasetLLADAV(Dataset): + + def __init__(self, samples: List[dict], method: str, lower: float, upper: float, block_size: int, mask_times: int, max_gen_length: int): + super().__init__() + self.samples = samples + self.method = method + self.lower = lower + self.upper = upper + self.block_size = block_size + self.mask_times = mask_times + self.max_gen_length = max_gen_length + + self.variants: List[dict] = [] + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt="\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=42) + for sid, s in enumerate(self.samples): + if self.method == "semi-ar": + step_map = s.get("step_map", None) + if step_map is None: + step_map = list(range(self.max_gen_length)) + order_list = collapse_k_unique(step_map, self.block_size) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(dict( + sample_id=sid, + prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt, + response=s["response"], + image_abs_path=s.get("image_abs_path", None), + image=s.get("image", None), + image_path=s.get("image_path", None), + video_abs_path=s.get("video_abs_path", None), + video=s.get("video", None), + video_path=s.get("video_path", None), + var_type="semi_ar", + var_param=int(st), + step_map=order_list, + mask_seed=None, + )) + elif self.method == "random_masking": + for _ in range(self.mask_times): + t = float(rng.uniform(self.lower, self.upper)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(dict( + sample_id=sid, + prompt=sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt, + response=s["response"], + image_abs_path=s.get("image_abs_path", None), + image=s.get("image", None), + image_path=s.get("image_path", None), + video_abs_path=s.get("video_abs_path", None), + video=s.get("video", None), + video_path=s.get("video_path", None), + var_type="random_masking", + var_param=t, + step_map=None, + mask_seed=mask_seed, + )) + else: + raise ValueError(f"Unknown SFT method: {self.method}") + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + v = self.variants[idx] + return { + "idx": idx, + "sample_id": v["sample_id"], + "prompt": v["prompt"], + "response": v["response"], + "image_abs_path": v.get("image_abs_path", None), + "image": v.get("image", None), + "image_path": v.get("image_path", None), + "video_abs_path": v.get("video_abs_path", None), + "video": v.get("video", None), + "video_path": v.get("video_path", None), + "var_type": v["var_type"], + "var_param": v["var_param"], + "step_map": v.get("step_map", None), + "mask_seed": v.get("mask_seed", None), + } + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "image": batch.get("image", [None])[first_pos], + "image_path": batch.get("image_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + "video": batch.get("video", [None])[first_pos], + "video_path": batch.get("video_path", [None])[first_pos], + } + + # 优先检测视频,其次检测图片 + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + + modalities = [media_type] + + # 若包含视频且需附加时间指引,将参数注入 prompt + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + # 剔除原有可能存在的图片 Token 以免重叠 + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + # Tokenize (内部会自动 prepend DEFAULT_IMAGE_TOKEN) + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) # (L, D) + new_labels = new_labels.squeeze(0) # (L,) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) # (M,) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [] + for i in range(bsz): + sid = int(batch["sample_id"][i]) + lengths.append(int(sample_cache[sid]["clean_embeds"].shape[0])) + Lmax = max(lengths) + D = next(iter(sample_cache.values()))["clean_embeds"].shape[1] + + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list = [] + labels_list = [] + p_mask_list = [] + attn_list = [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + step_map = batch["step_map"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + + if var_type == "semi_ar": + order = torch.as_tensor(step_map, device=device) + M = asst_idx.numel() + if order.numel() != M: + m = min(M, order.numel()) + asst_idx = asst_idx[:m] + order = order[:m] + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx] = order + step_val = int(var_param) + + p = torch.empty(L, device=device).uniform_(0.0, 1.0) + lower = 0.0 + 1e-6 + upper = 1.0 - 1e-6 + thresh = (upper - lower) * torch.rand(1, device=device) + lower + block_mask = (order_full == step_val) & (p < thresh) + + mask_pos = (order_full > step_val) | block_mask + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + pmask = block_mask + + + elif var_type == "random_masking": + M = asst_idx.numel() + if M == 0: + noisy_embeds = clean_embeds + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + gen = torch.Generator(device="cpu") + gen.manual_seed(int(mask_seed)) + + # 掷骰子判决 + rnd_cpu = torch.rand(M, generator=gen) < t + rnd = rnd_cpu.to(device=device) + + # 🌟 上帝之手:强行兜底逻辑 + # 如果概率太黑,导致 rnd 里面全是 False + if not rnd.any(): + # 在 0 到 M-1 之间随机挑一个位置 + rand_pos = torch.randint(0, M, (1,), device=device) + # 强行把这个位置变成 True + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + # elif var_type == "random_masking": + # M = asst_idx.numel() + # if M == 0: + # noisy_embeds = clean_embeds + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # else: + # t = float(var_param) + # mask_seed = batch.get("mask_seed", [None] * bsz)[i] + # if mask_seed is None: + # mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + # gen = torch.Generator(device="cpu") + # gen.manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + # rnd = rnd_cpu.to(device=device) + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + else: + continue + + if not pmask.any(): + if asst_idx.numel() > 0: + print("*****") + # 随机生成一个 0 到 M-1 的索引 + rand_pos = torch.randint(0, asst_idx.numel(), (1,), device=device) + force_idx = asst_idx[rand_pos] + + pmask[force_idx] = True + noisy_embeds[force_idx] = mask_embed + else: + print("*********bad data*********") + print(batch["sample_id"][i]) + continue + + # if not pmask.any(): + # # print('*****no token masked*****') + # if asst_idx.numel() > 0: + # # 强制 Mask 最后一个 assistant token + # force_idx = asst_idx[-1] + # pmask[force_idx] = True + # noisy_embeds[force_idx] = mask_embed + # else: + # print("*********bad data*********") + # print(batch["sample_id"][i]) + # # 如果连 assistant token 都没有,那这条数据本身就是坏的,只能跳过 + # continue + + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + return noisy_batch, labels_batch, p_mask_batch, attn_batch + +# ... 余下 save_checkpoint 和 main() 训练循环逻辑无需修改,保持原样即可 ... +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + # TF32 + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + + # 🌟 实例化 DDP 处理器,允许存在未使用的参数 + # ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True) + # process_group_kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=3600)) + + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with="wandb", + project_dir=config.experiment.logging_dir, + split_batches=True, + # kwargs_handlers=[process_group_kwargs] # 🌟 把超时设置传进去 + # kwargs_handlers=[ddp_kwargs], # <-- 把这个参数传进去 + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + run_id = config.wandb.get("run_id", None) + if run_id is None: + resume_wandb_run = False + run_id = wandb.util.generate_id() + config.wandb.run_id = run_id + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + pretrained_model = config.model.pretrained_model + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, LAVIDA_MODEL_NAME, attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + with open("./data/" + config.dataset.optimization_data + ".json", 'r') as f: + data_list = json.load(f) + + ds = SFTVariantDatasetLLADAV( + samples=data_list, + method=config.training.method, + lower=config.training.lower_p, + upper=config.training.upper_p, + block_size=config.training.block_size, + mask_times=config.training.mask_times_per_sample, + max_gen_length=config.training.max_gen_length + ) + + # Scheduler & DataLoader + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(ds) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + dl = DataLoader( + ds, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + drop_last=True, + ) + + model, optimizer, lr_scheduler, dl = accelerator.prepare(model, optimizer, lr_scheduler, dl) + + logger.info("***** Running LLaDA-V SFT training *****") + logger.info(f" Num variants = {len(ds)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + data_time_m = AverageMeter() + end = time.time() + + from tqdm.auto import tqdm + + def forward_process_batch(batch): + out = prepare_batch_noisy_embeds( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + return None + + noisy_embeds, labels, p_mask, attn = out + + outputs = model.get_model()( + inputs_embeds=noisy_embeds, + attention_mask=attn, + use_cache=False, + return_dict=True, + ) + logits = model.lm_head(outputs.last_hidden_state).float() # (B, L, V) + + # logits = outputs.logits.float() # (B, L, V) + log_probs = F.log_softmax(logits, dim=-1) + + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_tok = log_probs.gather(-1, safe_labels.unsqueeze(-1)).squeeze(-1) # (B, L) + + loss_tok = - (logp_tok * p_mask).sum(dim=1) + num_mask = torch.clamp(p_mask.sum(dim=1), min=1) + loss = (loss_tok / num_mask).mean() + return loss + + first_epoch = 0 + grad_accum = config.training.gradient_accumulation_steps + + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + dl, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + # for step, batch in enumerate(progress_bar, start=1): + # data_time_m.update(time.time() - end) + + # # 关键修复:使用 Accelerate 官方推荐的梯度累加上下文管理器 + # with accelerator.accumulate(model): + # # 不需要手动除以 grad_accum,Accelerate 会在底层自动处理 + # loss = forward_process_batch(batch) + + # accelerator.backward(loss) + + # # 只有当梯度累加达到步数,需要真实更新权重时,才做梯度裁剪 + # if accelerator.sync_gradients: + # if config.training.max_grad_norm is not None: + # accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + + # optimizer.step() + # lr_scheduler.step() + # optimizer.zero_grad(set_to_none=True) + + # # 警告:坚决删掉这里的 torch.cuda.empty_cache() + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + loss = forward_process_batch(batch) + if loss is None: + # loss = 0.0 * sum(p.sum() for p in model.parameters() if p.requires_grad) + continue + loss = loss / grad_accum + accelerator.backward(loss) + + if step % grad_accum == 0: + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + if step % grad_accum != 0: + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/wandb/run-20260529_013322-be4lg0e8/files/config.yaml b/wandb/run-20260529_013322-be4lg0e8/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9eeb14ac701a8e355abab544e6b2d1e560e6721 --- /dev/null +++ b/wandb/run-20260529_013322-be4lg0e8/files/config.yaml @@ -0,0 +1,176 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/sft_lavida_video_reversal.py + e: + fwjucqotvwyp2kdbu182tdcf5zycf271: + args: + - config=configs/sft_lavida_video_reversal.yaml + codePath: train/sft_lavida_video_reversal.py + codePathLocal: train/sft_lavida_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "623605473280" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/sft_lavida_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-05-28T17:33:22.212299Z" + writerId: fwjucqotvwyp2kdbu182tdcf5zycf271 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 13 + - 14 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/sft_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: video_reversal_sft_train_cleaned_5k +experiment.logging_dir: + value: sft_lavida_video_reversal/logs +experiment.num_node: + value: 1 +experiment.project: + value: sft_lavida_video_reversal +lr_scheduler.params.learning_rate: + value: 1e-05 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/hf/lavida-llada-v1.0-instruct +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-05 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +training.batch_size_lm: + value: 2 +training.block_size: + value: 16 +training.enable_tf32: + value: true +training.gradient_accumulation_steps: + value: 2 +training.gradient_checkpointing_enable: + value: true +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 512 +training.max_grad_norm: + value: 1 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.seed: + value: 10086 +training.upper_p: + value: 0.9 +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: be4lg0e8 diff --git a/wandb/run-20260529_013322-be4lg0e8/files/diff.patch b/wandb/run-20260529_013322-be4lg0e8/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..accd94971a45a35b1bccb2152d382aa875b91fd9 --- /dev/null +++ b/wandb/run-20260529_013322-be4lg0e8/files/diff.patch @@ -0,0 +1,321 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260529_013322-be4lg0e8/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260529_013322-be4lg0e8/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..accd94971a45a35b1bccb2152d382aa875b91fd9 --- /dev/null +++ b/wandb/run-20260529_013322-be4lg0e8/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,321 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) diff --git a/wandb/run-20260529_013322-be4lg0e8/files/output.log b/wandb/run-20260529_013322-be4lg0e8/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f8e4d908f02551c7f710cf02038682d50b7efab5 --- /dev/null +++ b/wandb/run-20260529_013322-be4lg0e8/files/output.log @@ -0,0 +1,51 @@ +05/29/2026 01:33:23 - INFO - root - Saving config to sft_lavida_video_reversal/config.yaml +Rank 0: Loaded LLaVA model: /data4/runqing/hf/lavida-llada-v1.0-instruct +05/29/2026 01:33:23 - INFO - llava.model.language_model.llada.modeling_llada - Initializing model parameters... +Rank 0: Loading vision tower: google/siglip-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 3.32s/it] +Rank 0: Model Class: LlavaLladaForMaskedDiffusion +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.71 GB Max_MA 16.71 GB CA 21.04 GB Max_CA 21 GB +CPU Virtual Memory: used = 118.75 GB, percent = 5.9% +After initializing optimizer states +MA 16.71 GB Max_MA 16.71 GB CA 21.04 GB Max_CA 21 GB +CPU Virtual Memory: used = 145.71 GB, percent = 7.2% +After initializing ZeRO optimizer +MA 16.71 GB Max_MA 16.71 GB CA 21.04 GB Max_CA 21 GB +CPU Virtual Memory: used = 145.85 GB, percent = 7.2% +05/29/2026 01:34:31 - INFO - __main__ - ***** Running LLaDA-V SFT training ***** +05/29/2026 01:34:31 - INFO - __main__ - Num variants = 10000 +05/29/2026 01:34:31 - INFO - __main__ - Num training steps = 626 +05/29/2026 01:34:31 - INFO - __main__ - Instantaneous batch size per device = 2 +05/29/2026 01:34:31 - INFO - __main__ - Total train batch size (parallel * accum) = 16 +05/29/2026 01:34:31 - INFO - __main__ - Gradient Accumulation steps = 2 +Epoch 1/1: 0%| | 0/1250 [00:00 + main() + File "/data4/runqing/dLLM-RL/train/sft_lavida_video_reversal.py", line 773, in main + loss = forward_process_batch(batch) + File "/data4/runqing/dLLM-RL/train/sft_lavida_video_reversal.py", line 715, in forward_process_batch + outputs = model.get_model()( + File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) +TypeError: LLaDAModel.forward() got an unexpected keyword argument 'inputs_embeds' +[rank0]: Traceback (most recent call last): +[rank0]: File "/data4/runqing/dLLM-RL/train/sft_lavida_video_reversal.py", line 802, in +[rank0]: main() +[rank0]: File "/data4/runqing/dLLM-RL/train/sft_lavida_video_reversal.py", line 773, in main +[rank0]: loss = forward_process_batch(batch) +[rank0]: File "/data4/runqing/dLLM-RL/train/sft_lavida_video_reversal.py", line 715, in forward_process_batch +[rank0]: outputs = model.get_model()( +[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: TypeError: LLaDAModel.forward() got an unexpected keyword argument 'inputs_embeds' diff --git a/wandb/run-20260529_013322-be4lg0e8/files/requirements.txt b/wandb/run-20260529_013322-be4lg0e8/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/wandb/run-20260529_013322-be4lg0e8/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260529_013322-be4lg0e8/files/wandb-metadata.json b/wandb/run-20260529_013322-be4lg0e8/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5e305beb8521681ba1774c6866c5d7c57e0967c5 --- /dev/null +++ b/wandb/run-20260529_013322-be4lg0e8/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-28T17:33:22.212299Z", + "args": [ + "config=configs/sft_lavida_video_reversal.yaml" + ], + "program": "/data4/runqing/dLLM-RL/train/sft_lavida_video_reversal.py", + "codePath": "train/sft_lavida_video_reversal.py", + "codePathLocal": "train/sft_lavida_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "623605473280" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "fwjucqotvwyp2kdbu182tdcf5zycf271" +} \ No newline at end of file diff --git a/wandb/run-20260529_013322-be4lg0e8/files/wandb-summary.json b/wandb/run-20260529_013322-be4lg0e8/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0df0a1ecf08f178bd5ef523c9f437abb61716860 --- /dev/null +++ b/wandb/run-20260529_013322-be4lg0e8/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":70},"_runtime":70} \ No newline at end of file diff --git a/wandb/run-20260529_013322-be4lg0e8/logs/debug-core.log b/wandb/run-20260529_013322-be4lg0e8/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..d7469dba3d866b13fb0e9516ad42e3be9f4d2c22 --- /dev/null +++ b/wandb/run-20260529_013322-be4lg0e8/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-05-29T01:33:22.328814213+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpizyr_7v8/port-3947030.txt","pid":3947030,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-29T01:33:22.329499322+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3947030} +{"time":"2026-05-29T01:33:22.329455045+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3947030-3947819-999903865/socket","Net":"unix"}} +{"time":"2026-05-29T01:33:22.479159437+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-29T01:33:22.488337064+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"be4lg0e8","id":"1(@)"} +{"time":"2026-05-29T01:33:23.002960594+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"be4lg0e8","id":"1(@)"} +{"time":"2026-05-29T01:33:28.25623034+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"9u6s3c9ykopq"} +{"time":"2026-05-29T01:34:33.571395483+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-29T01:34:33.571569119+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-29T01:34:33.571538974+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-29T01:34:33.571734829+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-29T01:34:33.571856489+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3947030-3947819-999903865/socket","Net":"unix"}} +{"time":"2026-05-29T01:34:33.690905851+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-29T01:34:33.690963914+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-29T01:34:33.690998676+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260529_013322-be4lg0e8/logs/debug-internal.log b/wandb/run-20260529_013322-be4lg0e8/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..11afc65fc2d06133bbc39acda76cc0cf1930da17 --- /dev/null +++ b/wandb/run-20260529_013322-be4lg0e8/logs/debug-internal.log @@ -0,0 +1,29 @@ +{"time":"2026-05-29T01:33:22.488475797+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-29T01:33:23.002265963+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-29T01:33:23.002406724+08:00","level":"INFO","msg":"stream: created new stream","id":"be4lg0e8"} +{"time":"2026-05-29T01:33:23.002787733+08:00","level":"INFO","msg":"handler: started","stream_id":"be4lg0e8"} +{"time":"2026-05-29T01:33:23.002939122+08:00","level":"INFO","msg":"stream: started","id":"be4lg0e8"} +{"time":"2026-05-29T01:33:23.002977332+08:00","level":"INFO","msg":"writer: started","stream_id":"be4lg0e8"} +{"time":"2026-05-29T01:33:23.002981329+08:00","level":"INFO","msg":"sender: started","stream_id":"be4lg0e8"} +{"time":"2026-05-29T01:33:23.004992991+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-29T01:33:28.211140376+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173328Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=35328c5d4edccc95bc5d0ddab44acf5de6e4e59cb08e667314f00bb679bb71a6"} +{"time":"2026-05-29T01:33:28.213088878+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173328Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=be93e73e5a24c5d5832d37168a023d4211444451b3f1eb0d51c3d6b2e656f769"} +{"time":"2026-05-29T01:33:28.213163302+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173328Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=57c33226b8335f9be325b82a6fd79e75cd8fb79f0fe6bdffab6ce7937906c970"} +{"time":"2026-05-29T01:33:28.213168239+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lavida_video_reversal/be4lg0e8/requirements.txtnetdisk/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/requirements.txt18B3CB1E2CDC0A9729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260529_013322-be4lg0e8/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173328Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=35328c5d4edccc95bc5d0ddab44acf5de6e4e59cb08e667314f00bb679bb71a6, Size: 2982}"} +{"time":"2026-05-29T01:33:28.213334198+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lavida_video_reversal/be4lg0e8/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18B3CB1E2CA3A14029678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260529_013322-be4lg0e8/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173328Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=be93e73e5a24c5d5832d37168a023d4211444451b3f1eb0d51c3d6b2e656f769, Size: 13158}"} +{"time":"2026-05-29T01:33:28.214383836+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lavida_video_reversal/be4lg0e8/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/wandb-metadata.json18B3CB1E2D91BEC54b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260529_013322-be4lg0e8/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173328Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=57c33226b8335f9be325b82a6fd79e75cd8fb79f0fe6bdffab6ce7937906c970, Size: 2650}"} +{"time":"2026-05-29T01:33:28.214897297+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/code/train/sft_lavida_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173328Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=81fd8f9f9988a6e7f27a0b85436b878fc0f1f4792dd7104e2dad9d8eca51387d"} +{"time":"2026-05-29T01:33:28.214989773+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lavida_video_reversal/be4lg0e8/code/train/sft_lavida_video_reversal.pynetdisk/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/code/train/sft_lavida_video_reversal.py18B3CB1E2D59FC894b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260529_013322-be4lg0e8/files/code/train/sft_lavida_video_reversal.py, Name: code/train/sft_lavida_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/code/train/sft_lavida_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173328Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=81fd8f9f9988a6e7f27a0b85436b878fc0f1f4792dd7104e2dad9d8eca51387d, Size: 31804}"} +{"time":"2026-05-29T01:33:28.22439176+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173328Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cc32204657e46d1fb1ce96fb397b41c863d81339aeb08f0ece059d0415919aa5"} +{"time":"2026-05-29T01:33:28.224449966+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lavida_video_reversal/be4lg0e8/diff.patchnetdisk/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/diff.patch18B3CB1E2CF8F96129678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260529_013322-be4lg0e8/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173328Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cc32204657e46d1fb1ce96fb397b41c863d81339aeb08f0ece059d0415919aa5, Size: 13158}"} +{"time":"2026-05-29T01:34:33.571573449+08:00","level":"INFO","msg":"stream: closing","id":"be4lg0e8"} +{"time":"2026-05-29T01:34:33.625848962+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173433Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=19d7908bc40c19f2e52e254720e289b7c2fcf5103e2e6cef03cc55d9b6f672a1"} +{"time":"2026-05-29T01:34:33.626012948+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lavida_video_reversal/be4lg0e8/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/wandb-summary.json18B3CB2D686F915917e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260529_013322-be4lg0e8/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173433Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=19d7908bc40c19f2e52e254720e289b7c2fcf5103e2e6cef03cc55d9b6f672a1, Size: 39}"} +{"time":"2026-05-29T01:34:33.627607925+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173433Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1f00de0c19432598d95aaef1f0284e975bde0d51dc2a070fda529d15a95eb818"} +{"time":"2026-05-29T01:34:33.627675178+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lavida_video_reversal/be4lg0e8/output.lognetdisk/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/output.log18B3CB2D68A64C554b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260529_013322-be4lg0e8/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173433Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1f00de0c19432598d95aaef1f0284e975bde0d51dc2a070fda529d15a95eb818, Size: 3804}"} +{"time":"2026-05-29T01:34:33.636916178+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173433Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=787c72f3f2d2375bfe9fd372fcec42693a0ffe62e12c7ab8ad96cf34ea026e88"} +{"time":"2026-05-29T01:34:33.637062829+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/sft_lavida_video_reversal/be4lg0e8/config.yamlnetdisk/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/config.yaml18B3CB2D6888821529678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260529_013322-be4lg0e8/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/sft_lavida_video_reversal/be4lg0e8/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260528%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260528T173433Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=787c72f3f2d2375bfe9fd372fcec42693a0ffe62e12c7ab8ad96cf34ea026e88, Size: 5616}"} +{"time":"2026-05-29T01:34:33.638112469+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-29T01:34:33.690068106+08:00","level":"INFO","msg":"handler: closed","stream_id":"be4lg0e8"} +{"time":"2026-05-29T01:34:33.690289355+08:00","level":"INFO","msg":"sender: closed","stream_id":"be4lg0e8"} +{"time":"2026-05-29T01:34:33.690320316+08:00","level":"INFO","msg":"stream: closed","id":"be4lg0e8"} diff --git a/wandb/run-20260529_013322-be4lg0e8/logs/debug.log b/wandb/run-20260529_013322-be4lg0e8/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ef5bfaebc25a3e08cc3a3e4b4c69ca3b65db6ba8 --- /dev/null +++ b/wandb/run-20260529_013322-be4lg0e8/logs/debug.log @@ -0,0 +1,22 @@ +2026-05-29 01:33:22,214 INFO MainThread:3947030 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-29 01:33:22,214 INFO MainThread:3947030 [wandb_setup.py:_flush():81] Configure stats pid to 3947030 +2026-05-29 01:33:22,214 INFO MainThread:3947030 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-29 01:33:22,214 INFO MainThread:3947030 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260529_013322-be4lg0e8/logs/debug.log +2026-05-29 01:33:22,214 INFO MainThread:3947030 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260529_013322-be4lg0e8/logs/debug-internal.log +2026-05-29 01:33:22,214 INFO MainThread:3947030 [wandb_init.py:init():844] calling init triggers +2026-05-29 01:33:22,214 INFO MainThread:3947030 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/sft_lavida_video_reversal.py'}} +2026-05-29 01:33:22,214 INFO MainThread:3947030 [wandb_init.py:init():892] starting backend +2026-05-29 01:33:22,479 INFO MainThread:3947030 [wandb_init.py:init():895] sending inform_init request +2026-05-29 01:33:22,482 INFO MainThread:3947030 [wandb_init.py:init():903] backend started and connected +2026-05-29 01:33:22,485 INFO MainThread:3947030 [wandb_init.py:init():973] updated telemetry +2026-05-29 01:33:22,491 INFO MainThread:3947030 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-29 01:33:23,131 INFO MainThread:3947030 [wandb_init.py:init():1042] starting run threads in backend +2026-05-29 01:33:23,251 INFO MainThread:3947030 [wandb_run.py:_console_start():2524] atexit reg +2026-05-29 01:33:23,251 INFO MainThread:3947030 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-29 01:33:23,251 INFO MainThread:3947030 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-29 01:33:23,251 INFO MainThread:3947030 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-29 01:33:23,255 INFO MainThread:3947030 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-29 01:33:23,256 INFO MainThread:3947030 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'be4lg0e8', 'experiment.project': 'sft_lavida_video_reversal', 'experiment.num_node': 1, 'experiment.logging_dir': 'sft_lavida_video_reversal/logs', 'model.pretrained_model': '/data4/runqing/hf/lavida-llada-v1.0-instruct', 'model.optimized_name': 'optimized', 'dataset.optimization_data': 'video_reversal_sft_train_cleaned_5k', 'dataset.data_type': 'video_reverse', 'training.gradient_checkpointing_enable': True, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.block_size': 16, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 512, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-05, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-05, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'config': 'configs/sft_lavida_video_reversal.yaml'} +2026-05-29 01:34:33,571 INFO wandb-AsyncioManager-main:3947030 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-29 01:34:33,571 INFO wandb-AsyncioManager-main:3947030 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/wandb/run-20260529_013322-be4lg0e8/run-be4lg0e8.wandb b/wandb/run-20260529_013322-be4lg0e8/run-be4lg0e8.wandb new file mode 100644 index 0000000000000000000000000000000000000000..82515fc5c3bab6b7ccdf1e4d935d1f135db90dc7 Binary files /dev/null and b/wandb/run-20260529_013322-be4lg0e8/run-be4lg0e8.wandb differ diff --git a/wandb/run-20260530_181727-fmmkx1a5/files/code/train/rl_lavida.py b/wandb/run-20260530_181727-fmmkx1a5/files/code/train/rl_lavida.py new file mode 100644 index 0000000000000000000000000000000000000000..3e111f99f33455c8199e45d6e289cc4dfee34b6d --- /dev/null +++ b/wandb/run-20260530_181727-fmmkx1a5/files/code/train/rl_lavida.py @@ -0,0 +1,840 @@ +import os +import sys +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LAVIDA_ROOT = os.path.join(REPO_ROOT, "LaViDa") +sys.path.insert(0, LAVIDA_ROOT) +sys.path.insert(1, REPO_ROOT) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter, ensure_wandb_run_id +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + +LAVIDA_MODEL_NAME = "llava_llada" + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()( + input_ids=None, + input_embeddings=noisy_embeds, + attention_mask=attn_batch, + use_cache=False, + ) + logits = outputs.logits.float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + existing_run_id = config.wandb.get("run_id", None) + run_id = ensure_wandb_run_id(config) + if existing_run_id is None: + resume_wandb_run = False + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LaViDa model and tokenizer from official LaViDa codebase") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, LAVIDA_MODEL_NAME, attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + noisy_embeds.requires_grad_(True) + outputs = model.get_model()( + input_ids=None, + input_embeddings=noisy_embeds, + attention_mask=attn_mask, + use_cache=False, + ) + logits = outputs.logits.float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/wandb/run-20260530_181727-fmmkx1a5/files/config.yaml b/wandb/run-20260530_181727-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1d292c18eb2f374114688aa273513ca55c73e6e --- /dev/null +++ b/wandb/run-20260530_181727-fmmkx1a5/files/config.yaml @@ -0,0 +1,288 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lavida.py + e: + d2vg153bfwmxihif84lw6c661901vfgf: + args: + - config=configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.current_epoch=21 + codePath: train/rl_lavida.py + codePathLocal: train/rl_lavida.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625595944960" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lavida.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-05-30T10:17:27.785406Z" + writerId: d2vg153bfwmxihif84lw6c661901vfgf + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 5 + - 13 + - 14 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 21 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/wandb/run-20260530_181727-fmmkx1a5/files/diff.patch b/wandb/run-20260530_181727-fmmkx1a5/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260530_181727-fmmkx1a5/files/diff.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260530_181727-fmmkx1a5/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260530_181727-fmmkx1a5/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260530_181727-fmmkx1a5/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260530_181727-fmmkx1a5/files/output.log b/wandb/run-20260530_181727-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..51b670b5aecfe686377928b50b0d4e1aec338d41 --- /dev/null +++ b/wandb/run-20260530_181727-fmmkx1a5/files/output.log @@ -0,0 +1,65 @@ +05/30/2026 18:17:28 - INFO - root - Saving config to rl_lavida_video_reversal/config.yaml +05/30/2026 18:17:28 - INFO - __main__ - Loading LaViDa model and tokenizer from official LaViDa codebase +Rank 0: Loaded LLaVA model: ./rl_lavida_video_reversal/ckpt/optimized +05/30/2026 18:17:28 - INFO - llava.model.language_model.llada.modeling_llada - Initializing model parameters... +Rank 0: Loading vision tower: google/siglip-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:03<00:00, 1.06it/s] +Rank 0: Restored mismatched LLaDA input embeddings from checkpoint shards. +Rank 0: Model Class: LlavaLladaForMaskedDiffusion +05/30/2026 18:17:36 - INFO - __main__ - Preprocessing Data +05/30/2026 18:17:36 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.68 GB Max_MA 16.68 GB CA 20.05 GB Max_CA 20 GB +CPU Virtual Memory: used = 107.46 GB, percent = 5.3% +After initializing optimizer states +MA 16.68 GB Max_MA 16.68 GB CA 20.05 GB Max_CA 20 GB +CPU Virtual Memory: used = 130.54 GB, percent = 6.5% +After initializing ZeRO optimizer +MA 16.68 GB Max_MA 16.68 GB CA 20.05 GB Max_CA 20 GB +CPU Virtual Memory: used = 131.0 GB, percent = 6.5% +05/30/2026 18:18:17 - INFO - __main__ - ***** Running inference ***** +05/30/2026 18:18:17 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +05/30/2026 18:22:44 - INFO - __main__ - ***** Running training ***** +05/30/2026 18:22:44 - INFO - __main__ - Num variants = 256 +05/30/2026 18:22:44 - INFO - __main__ - Num training steps = 33 +05/30/2026 18:22:44 - INFO - __main__ - Instantaneous batch size per device = 1 +05/30/2026 18:22:44 - INFO - __main__ - Total train batch size (parallel * accum) = 8 +05/30/2026 18:22:44 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 3%|▎ | 2/64 [00:52<25:46, 24.94s/it]wandb: WARNING Tried to log to step 1 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 6%|▋ | 4/64 [01:26<19:25, 19.43s/it]wandb: WARNING Tried to log to step 2 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 9%|▉ | 6/64 [01:59<17:08, 17.73s/it]wandb: WARNING Tried to log to step 3 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 12%|█▎ | 8/64 [02:33<16:14, 17.40s/it]wandb: WARNING Tried to log to step 4 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 16%|█▌ | 10/64 [03:07<15:33, 17.28s/it]wandb: WARNING Tried to log to step 5 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 19%|█▉ | 12/64 [03:41<14:45, 17.03s/it]wandb: WARNING Tried to log to step 6 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 22%|██▏ | 14/64 [04:05<12:12, 14.64s/it]wandb: WARNING Tried to log to step 7 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 25%|██▌ | 16/64 [04:29<10:42, 13.38s/it]wandb: WARNING Tried to log to step 8 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 28%|██▊ | 18/64 [04:52<09:33, 12.46s/it]wandb: WARNING Tried to log to step 9 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 31%|███▏ | 20/64 [05:18<09:19, 12.72s/it]wandb: WARNING Tried to log to step 10 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 34%|███▍ | 22/64 [05:40<08:21, 11.95s/it]wandb: WARNING Tried to log to step 11 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 38%|███▊ | 24/64 [06:03<07:44, 11.61s/it]wandb: WARNING Tried to log to step 12 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 41%|████ | 26/64 [06:44<10:07, 16.00s/it]wandb: WARNING Tried to log to step 13 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 44%|████▍ | 28/64 [07:22<10:34, 17.62s/it]wandb: WARNING Tried to log to step 14 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 47%|████▋ | 30/64 [07:58<10:14, 18.09s/it]wandb: WARNING Tried to log to step 15 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 50%|█████ | 32/64 [08:32<09:22, 17.58s/it]wandb: WARNING Tried to log to step 16 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 53%|█████▎ | 34/64 [09:05<08:34, 17.14s/it]wandb: WARNING Tried to log to step 17 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 56%|█████▋ | 36/64 [09:41<08:08, 17.45s/it]wandb: WARNING Tried to log to step 18 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 59%|█████▉ | 38/64 [10:16<07:35, 17.51s/it]wandb: WARNING Tried to log to step 19 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 62%|██████▎ | 40/64 [10:54<07:17, 18.21s/it]wandb: WARNING Tried to log to step 20 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 66%|██████▌ | 42/64 [11:17<05:27, 14.91s/it]wandb: WARNING Tried to log to step 21 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 69%|██████▉ | 44/64 [11:39<04:21, 13.08s/it]wandb: WARNING Tried to log to step 22 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 72%|███████▏ | 46/64 [12:16<04:39, 15.55s/it]wandb: WARNING Tried to log to step 23 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 75%|███████▌ | 48/64 [12:49<04:18, 16.13s/it]wandb: WARNING Tried to log to step 24 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 78%|███████▊ | 50/64 [13:25<03:59, 17.08s/it]wandb: WARNING Tried to log to step 25 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 81%|████████▏ | 52/64 [13:59<03:25, 17.14s/it]wandb: WARNING Tried to log to step 26 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 84%|████████▍ | 54/64 [14:21<02:21, 14.11s/it]wandb: WARNING Tried to log to step 27 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 88%|████████▊ | 56/64 [14:46<01:45, 13.19s/it]wandb: WARNING Tried to log to step 28 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 91%|█████████ | 58/64 [15:09<01:14, 12.49s/it]wandb: WARNING Tried to log to step 29 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 94%|█████████▍| 60/64 [15:35<00:50, 12.75s/it]wandb: WARNING Tried to log to step 30 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 97%|█████████▋| 62/64 [15:57<00:23, 11.81s/it]wandb: WARNING Tried to log to step 31 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 100%|██████████| 64/64 [16:19<00:00, 15.30s/it] +wandb: WARNING Tried to log to step 32 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +05/30/2026 18:39:41 - INFO - __main__ - Saved model + tokenizer to rl_lavida_video_reversal/ckpt/optimized diff --git a/wandb/run-20260530_181727-fmmkx1a5/files/requirements.txt b/wandb/run-20260530_181727-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/wandb/run-20260530_181727-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260530_181727-fmmkx1a5/files/wandb-metadata.json b/wandb/run-20260530_181727-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ca4bf75e4fe0e00e343d60b81bbb09870c6d0ec0 --- /dev/null +++ b/wandb/run-20260530_181727-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,94 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-30T10:17:27.785406Z", + "args": [ + "config=configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.current_epoch=21" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lavida.py", + "codePath": "train/rl_lavida.py", + "codePathLocal": "train/rl_lavida.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625595944960" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "d2vg153bfwmxihif84lw6c661901vfgf" +} \ No newline at end of file diff --git a/wandb/run-20260530_181727-fmmkx1a5/files/wandb-summary.json b/wandb/run-20260530_181727-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d1fc7827e3ff6a407ebbc74746096022854ce3bc --- /dev/null +++ b/wandb/run-20260530_181727-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/acc":0.396484375,"reward/avg_response_length":124.568359375,"reward/t_grpo_bonus":0.2718750000000003,"_runtime":19367,"reward/unbiased_bonus":0.09374999999999996,"train/learning_rate":8.997213817017502e-07,"train/loss":-0.3729729354381562,"reward/train_step":1,"_wandb":{"runtime":19367},"_step":38,"_timestamp":1.7801299761845806e+09} \ No newline at end of file diff --git a/wandb/run-20260530_181727-fmmkx1a5/logs/debug-core.log b/wandb/run-20260530_181727-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..95692432c3b808f54a9c71b4d95331d044daca2d --- /dev/null +++ b/wandb/run-20260530_181727-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-05-30T18:17:27.854274532+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp8d2f922u/port-3656239.txt","pid":3656239,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T18:17:27.854981799+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3656239} +{"time":"2026-05-30T18:17:27.854942454+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3656239-3656983-2371143700/socket","Net":"unix"}} +{"time":"2026-05-30T18:17:28.035582506+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T18:17:28.042119306+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T18:17:28.336992311+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T18:17:33.664449159+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fbjifbrkgmk3"} +{"time":"2026-05-30T18:39:42.398519492+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fbjifbrkgmk3"} +{"time":"2026-05-30T18:39:42.594954852+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fbjifbrkgmk3"} +{"time":"2026-05-30T18:39:42.595796027+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T18:39:42.596676609+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T18:39:43.402358025+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T18:39:43.402395422+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T18:39:43.402411016+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T18:39:43.402420797+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T18:39:43.402526156+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3656239-3656983-2371143700/socket","Net":"unix"}} +{"time":"2026-05-30T18:39:43.402547988+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T18:39:43.402609933+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T18:39:43.402635504+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260530_181727-fmmkx1a5/logs/debug-internal.log b/wandb/run-20260530_181727-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d35b086286b191ba91e284e754fe2f60ef2531ec --- /dev/null +++ b/wandb/run-20260530_181727-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,62 @@ +{"time":"2026-05-30T18:17:28.042204273+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T18:17:28.336521304+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T18:17:28.336694955+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T18:17:28.336780918+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T18:17:28.336943207+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T18:17:28.336950082+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T18:17:28.336992297+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T18:17:28.34182633+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T18:17:33.62251184+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d0932c55f79458da3709fc97838274a892d3fd52bf6178bb060bb1aab2f8b138"} +{"time":"2026-05-30T18:17:33.622651742+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B4507DB91F9D4029678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_181727-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d0932c55f79458da3709fc97838274a892d3fd52bf6178bb060bb1aab2f8b138, Size: 2665}"} +{"time":"2026-05-30T18:17:33.623938525+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=380dbd38e38c10dfe06b40ff79357fd90576fd3f389fea96d8b9ec0d8bf8a17d"} +{"time":"2026-05-30T18:17:33.624070711+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B4507DB783CDDF17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_181727-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=380dbd38e38c10dfe06b40ff79357fd90576fd3f389fea96d8b9ec0d8bf8a17d, Size: 2982}"} +{"time":"2026-05-30T18:17:33.630661781+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=be968850e391ca367c9e76fec87ee40eab3952afd9bdf1e8334fc5ee3800f45f"} +{"time":"2026-05-30T18:17:33.630815908+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18B4507DB7F0D68E4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_181727-fmmkx1a5/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=be968850e391ca367c9e76fec87ee40eab3952afd9bdf1e8334fc5ee3800f45f, Size: 14311}"} +{"time":"2026-05-30T18:17:33.631172402+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=132a725ab6ff666b081067f736ff770cb051273fdd47477ebdfd32119b2221d9"} +{"time":"2026-05-30T18:17:33.631209282+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff.patchnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff.patch18B4507DB9252AFB29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_181727-fmmkx1a5/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=132a725ab6ff666b081067f736ff770cb051273fdd47477ebdfd32119b2221d9, Size: 14311}"} +{"time":"2026-05-30T18:17:33.632857317+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/train/rl_lavida.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=418507b8ade064244fa40eefe97f85628d71a948f9718d4df9b2903301709f2b"} +{"time":"2026-05-30T18:17:33.632938844+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/train/rl_lavida.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/train/rl_lavida.py18B4507DB93135BF29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_181727-fmmkx1a5/files/code/train/rl_lavida.py, Name: code/train/rl_lavida.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/train/rl_lavida.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=418507b8ade064244fa40eefe97f85628d71a948f9718d4df9b2903301709f2b, Size: 32927}"} +{"time":"2026-05-30T18:23:37.464824925+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":1,"current":39} +{"time":"2026-05-30T18:24:11.049072751+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":2,"current":39} +{"time":"2026-05-30T18:24:43.946424368+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":3,"current":39} +{"time":"2026-05-30T18:25:18.12037505+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":4,"current":39} +{"time":"2026-05-30T18:25:52.822242245+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":5,"current":39} +{"time":"2026-05-30T18:26:26.085082886+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":6,"current":39} +{"time":"2026-05-30T18:26:50.44106953+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":7,"current":39} +{"time":"2026-05-30T18:27:14.621023609+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":8,"current":39} +{"time":"2026-05-30T18:27:37.732713309+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":9,"current":39} +{"time":"2026-05-30T18:28:03.14366178+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":10,"current":39} +{"time":"2026-05-30T18:28:25.714715476+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":11,"current":39} +{"time":"2026-05-30T18:28:48.285727378+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":12,"current":39} +{"time":"2026-05-30T18:29:29.015461292+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":13,"current":39} +{"time":"2026-05-30T18:30:07.112877513+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":14,"current":39} +{"time":"2026-05-30T18:30:43.497782466+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":15,"current":39} +{"time":"2026-05-30T18:31:17.620663195+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":16,"current":39} +{"time":"2026-05-30T18:31:50.72833093+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":17,"current":39} +{"time":"2026-05-30T18:32:26.448538867+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":18,"current":39} +{"time":"2026-05-30T18:33:01.671849952+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":19,"current":39} +{"time":"2026-05-30T18:33:39.157046333+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":20,"current":39} +{"time":"2026-05-30T18:34:02.433767282+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":21,"current":39} +{"time":"2026-05-30T18:34:24.746840191+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":22,"current":39} +{"time":"2026-05-30T18:35:00.899709339+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":23,"current":39} +{"time":"2026-05-30T18:35:34.371845243+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":24,"current":39} +{"time":"2026-05-30T18:36:10.053762173+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":25,"current":39} +{"time":"2026-05-30T18:36:44.131333762+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":26,"current":39} +{"time":"2026-05-30T18:37:06.228349201+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":27,"current":39} +{"time":"2026-05-30T18:37:30.944204779+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":28,"current":39} +{"time":"2026-05-30T18:37:54.581530192+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":29,"current":39} +{"time":"2026-05-30T18:38:19.999378079+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":30,"current":39} +{"time":"2026-05-30T18:38:41.885149105+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":31,"current":39} +{"time":"2026-05-30T18:39:03.881582072+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":32,"current":39} +{"time":"2026-05-30T18:39:42.547776855+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T103942Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9d87ef6443e483cbe98c8687294908db6db87c807c3d81f709edb5a6c2ca891d"} +{"time":"2026-05-30T18:39:42.547956958+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B451B321856C8017e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_181727-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T103942Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9d87ef6443e483cbe98c8687294908db6db87c807c3d81f709edb5a6c2ca891d, Size: 7965}"} +{"time":"2026-05-30T18:39:42.548758132+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T103942Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c4591e581a27783828b18f3611a50c68b0359d7185a9bb35a8f00f5ef35d1bdd"} +{"time":"2026-05-30T18:39:42.548881748+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B451B3220CDCE14b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_181727-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T103942Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c4591e581a27783828b18f3611a50c68b0359d7185a9bb35a8f00f5ef35d1bdd, Size: 344}"} +{"time":"2026-05-30T18:39:42.550234836+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T103942Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=bb47a87ccf770cfb1542f6f9b679985793fa276421ea9505c15ec8501a457059"} +{"time":"2026-05-30T18:39:42.550371427+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B451B32216428E4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_181727-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T103942Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=bb47a87ccf770cfb1542f6f9b679985793fa276421ea9505c15ec8501a457059, Size: 11230}"} +{"time":"2026-05-30T18:39:42.551352228+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T18:39:42.588561281+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T18:39:42.595813883+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T18:39:42.595827395+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T18:39:42.595896541+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T18:39:42.595910448+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/wandb/run-20260530_181727-fmmkx1a5/logs/debug.log b/wandb/run-20260530_181727-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..49070f2780825c07f971ccaeee45de657e2d18fe --- /dev/null +++ b/wandb/run-20260530_181727-fmmkx1a5/logs/debug.log @@ -0,0 +1,26 @@ +2026-05-30 18:17:27,788 INFO MainThread:3656239 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 18:17:27,788 INFO MainThread:3656239 [wandb_setup.py:_flush():81] Configure stats pid to 3656239 +2026-05-30 18:17:27,788 INFO MainThread:3656239 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 18:17:27,788 INFO MainThread:3656239 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260530_181727-fmmkx1a5/logs/debug.log +2026-05-30 18:17:27,788 INFO MainThread:3656239 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260530_181727-fmmkx1a5/logs/debug-internal.log +2026-05-30 18:17:27,788 INFO MainThread:3656239 [wandb_init.py:init():844] calling init triggers +2026-05-30 18:17:27,788 INFO MainThread:3656239 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lavida.py'}} +2026-05-30 18:17:27,788 INFO MainThread:3656239 [wandb_init.py:init():892] starting backend +2026-05-30 18:17:28,036 INFO MainThread:3656239 [wandb_init.py:init():895] sending inform_init request +2026-05-30 18:17:28,039 INFO MainThread:3656239 [wandb_init.py:init():903] backend started and connected +2026-05-30 18:17:28,044 INFO MainThread:3656239 [wandb_init.py:init():973] updated telemetry +2026-05-30 18:17:28,050 INFO MainThread:3656239 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 18:17:28,544 INFO MainThread:3656239 [wandb_init.py:init():1037] run resumed +2026-05-30 18:17:28,548 INFO MainThread:3656239 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 18:17:28,658 INFO MainThread:3656239 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 18:17:28,658 INFO MainThread:3656239 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 18:17:28,658 INFO MainThread:3656239 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 18:17:28,658 INFO MainThread:3656239 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 18:17:28,663 INFO MainThread:3656239 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 18:17:28,663 INFO MainThread:3656239 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 21, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lavida_video_reversal/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lavida_video_reversal.yaml'} +2026-05-30 18:39:42,397 INFO MainThread:3656239 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 18:39:42,397 INFO MainThread:3656239 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 18:39:42,398 INFO MainThread:3656239 [wandb_run.py:_restore():2472] restore +2026-05-30 18:39:42,398 INFO MainThread:3656239 [wandb_run.py:_restore():2478] restore done +2026-05-30 18:39:42,595 INFO MainThread:3656239 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260530_233437-fmmkx1a5/files/code/train/rl_lavida.py b/wandb/run-20260530_233437-fmmkx1a5/files/code/train/rl_lavida.py new file mode 100644 index 0000000000000000000000000000000000000000..3e111f99f33455c8199e45d6e289cc4dfee34b6d --- /dev/null +++ b/wandb/run-20260530_233437-fmmkx1a5/files/code/train/rl_lavida.py @@ -0,0 +1,840 @@ +import os +import sys +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LAVIDA_ROOT = os.path.join(REPO_ROOT, "LaViDa") +sys.path.insert(0, LAVIDA_ROOT) +sys.path.insert(1, REPO_ROOT) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter, ensure_wandb_run_id +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + +LAVIDA_MODEL_NAME = "llava_llada" + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()( + input_ids=None, + input_embeddings=noisy_embeds, + attention_mask=attn_batch, + use_cache=False, + ) + logits = outputs.logits.float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + existing_run_id = config.wandb.get("run_id", None) + run_id = ensure_wandb_run_id(config) + if existing_run_id is None: + resume_wandb_run = False + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LaViDa model and tokenizer from official LaViDa codebase") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, LAVIDA_MODEL_NAME, attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + noisy_embeds.requires_grad_(True) + outputs = model.get_model()( + input_ids=None, + input_embeddings=noisy_embeds, + attention_mask=attn_mask, + use_cache=False, + ) + logits = outputs.logits.float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/wandb/run-20260530_233437-fmmkx1a5/files/config.yaml b/wandb/run-20260530_233437-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..733f034feef85f94cfbaa3f24da645aaea1e4b71 --- /dev/null +++ b/wandb/run-20260530_233437-fmmkx1a5/files/config.yaml @@ -0,0 +1,288 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lavida.py + e: + clq968nuj6mayttuta6ihyrzizlvcdxi: + args: + - config=configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.current_epoch=26 + codePath: train/rl_lavida.py + codePathLocal: train/rl_lavida.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625600155648" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lavida.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-05-30T15:34:37.315196Z" + writerId: clq968nuj6mayttuta6ihyrzizlvcdxi + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 5 + - 13 + - 14 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 26 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/wandb/run-20260530_233437-fmmkx1a5/files/diff.patch b/wandb/run-20260530_233437-fmmkx1a5/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260530_233437-fmmkx1a5/files/diff.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260530_233437-fmmkx1a5/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260530_233437-fmmkx1a5/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260530_233437-fmmkx1a5/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260530_233437-fmmkx1a5/files/output.log b/wandb/run-20260530_233437-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0e2b6ebe76d76470ac935d39c6b8896d356b83c8 --- /dev/null +++ b/wandb/run-20260530_233437-fmmkx1a5/files/output.log @@ -0,0 +1,61 @@ +05/30/2026 23:34:38 - INFO - root - Saving config to rl_lavida_video_reversal/config.yaml +05/30/2026 23:34:38 - INFO - __main__ - Loading LaViDa model and tokenizer from official LaViDa codebase +Rank 0: Loaded LLaVA model: ./rl_lavida_video_reversal/ckpt/optimized +05/30/2026 23:34:38 - INFO - llava.model.language_model.llada.modeling_llada - Initializing model parameters... +Rank 0: Loading vision tower: google/siglip-so400m-patch14-384 +Loading checkpoint shards: 100%|██████████| 4/4 [00:03<00:00, 1.03it/s] +Rank 0: Restored mismatched LLaDA input embeddings from checkpoint shards. +Rank 0: Model Class: LlavaLladaForMaskedDiffusion +05/30/2026 23:34:45 - INFO - __main__ - Preprocessing Data +05/30/2026 23:34:46 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.68 GB Max_MA 16.68 GB CA 20.05 GB Max_CA 20 GB +CPU Virtual Memory: used = 110.73 GB, percent = 5.5% +After initializing optimizer states +MA 16.68 GB Max_MA 16.68 GB CA 20.05 GB Max_CA 20 GB +CPU Virtual Memory: used = 136.27 GB, percent = 6.8% +After initializing ZeRO optimizer +MA 16.68 GB Max_MA 16.68 GB CA 20.05 GB Max_CA 20 GB +CPU Virtual Memory: used = 136.27 GB, percent = 6.8% +05/30/2026 23:35:26 - INFO - __main__ - ***** Running inference ***** +05/30/2026 23:35:26 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +05/30/2026 23:38:34 - INFO - __main__ - ***** Running training ***** +05/30/2026 23:38:34 - INFO - __main__ - Num variants = 224 +05/30/2026 23:38:34 - INFO - __main__ - Num training steps = 29 +05/30/2026 23:38:34 - INFO - __main__ - Instantaneous batch size per device = 1 +05/30/2026 23:38:34 - INFO - __main__ - Total train batch size (parallel * accum) = 8 +05/30/2026 23:38:34 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 4%|▎ | 2/56 [00:44<18:50, 20.94s/it]wandb: WARNING Tried to log to step 1 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 7%|▋ | 4/56 [01:09<13:18, 15.35s/it]wandb: WARNING Tried to log to step 2 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 11%|█ | 6/56 [01:31<10:32, 12.65s/it]wandb: WARNING Tried to log to step 3 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 14%|█▍ | 8/56 [01:51<09:00, 11.25s/it]wandb: WARNING Tried to log to step 4 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 18%|█▊ | 10/56 [02:26<11:05, 14.47s/it]wandb: WARNING Tried to log to step 5 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 21%|██▏ | 12/56 [02:58<11:07, 15.18s/it]wandb: WARNING Tried to log to step 6 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 25%|██▌ | 14/56 [03:31<11:15, 16.08s/it]wandb: WARNING Tried to log to step 7 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 29%|██▊ | 16/56 [04:04<10:56, 16.41s/it]wandb: WARNING Tried to log to step 8 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 32%|███▏ | 18/56 [04:28<09:04, 14.33s/it]wandb: WARNING Tried to log to step 9 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 36%|███▌ | 20/56 [04:57<08:37, 14.36s/it]wandb: WARNING Tried to log to step 10 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 39%|███▉ | 22/56 [05:30<08:44, 15.41s/it]wandb: WARNING Tried to log to step 11 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 43%|████▎ | 24/56 [06:03<08:34, 16.08s/it]wandb: WARNING Tried to log to step 12 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 46%|████▋ | 26/56 [06:26<06:53, 13.78s/it]wandb: WARNING Tried to log to step 13 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 50%|█████ | 28/56 [06:48<05:43, 12.26s/it]wandb: WARNING Tried to log to step 14 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 54%|█████▎ | 30/56 [07:25<06:43, 15.53s/it]wandb: WARNING Tried to log to step 15 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 57%|█████▋ | 32/56 [08:05<07:14, 18.11s/it]wandb: WARNING Tried to log to step 16 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 61%|██████ | 34/56 [08:31<05:47, 15.78s/it]wandb: WARNING Tried to log to step 17 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 64%|██████▍ | 36/56 [08:59<04:58, 14.93s/it]wandb: WARNING Tried to log to step 18 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 68%|██████▊ | 38/56 [09:21<03:57, 13.20s/it]wandb: WARNING Tried to log to step 19 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 71%|███████▏ | 40/56 [09:48<03:33, 13.37s/it]wandb: WARNING Tried to log to step 20 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 75%|███████▌ | 42/56 [10:11<02:50, 12.16s/it]wandb: WARNING Tried to log to step 21 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 79%|███████▊ | 44/56 [10:35<02:27, 12.26s/it]wandb: WARNING Tried to log to step 22 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 82%|████████▏ | 46/56 [11:10<02:28, 14.86s/it]wandb: WARNING Tried to log to step 23 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 86%|████████▌ | 48/56 [11:43<02:07, 15.94s/it]wandb: WARNING Tried to log to step 24 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 89%|████████▉ | 50/56 [12:08<01:25, 14.22s/it]wandb: WARNING Tried to log to step 25 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 93%|█████████▎| 52/56 [12:34<00:53, 13.39s/it]wandb: WARNING Tried to log to step 26 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 96%|█████████▋| 54/56 [13:09<00:31, 15.80s/it]wandb: WARNING Tried to log to step 27 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 100%|██████████| 56/56 [13:48<00:00, 14.80s/it] +wandb: WARNING Tried to log to step 28 that is less than the current step 39. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +05/30/2026 23:53:02 - INFO - __main__ - Saved model + tokenizer to rl_lavida_video_reversal/ckpt/optimized diff --git a/wandb/run-20260530_233437-fmmkx1a5/files/requirements.txt b/wandb/run-20260530_233437-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/wandb/run-20260530_233437-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260530_233437-fmmkx1a5/files/wandb-metadata.json b/wandb/run-20260530_233437-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6db597f19ee79eb70816555059b3400333dbd4b6 --- /dev/null +++ b/wandb/run-20260530_233437-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,94 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-30T15:34:37.315196Z", + "args": [ + "config=configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.current_epoch=26" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lavida.py", + "codePath": "train/rl_lavida.py", + "codePathLocal": "train/rl_lavida.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625600155648" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "clq968nuj6mayttuta6ihyrzizlvcdxi" +} \ No newline at end of file diff --git a/wandb/run-20260530_233437-fmmkx1a5/files/wandb-summary.json b/wandb/run-20260530_233437-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7b1a38b3e33d27bf69aa7858066fa21574424b74 --- /dev/null +++ b/wandb/run-20260530_233437-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/t_grpo_bonus":0.2718750000000003,"_timestamp":1.7801299761845806e+09,"train/loss":-0.3729729354381562,"train/learning_rate":8.997213817017502e-07,"_wandb":{"runtime":24255},"_runtime":24255,"reward/unbiased_bonus":0.09374999999999996,"reward/acc":0.396484375,"reward/train_step":1,"_step":38,"reward/avg_response_length":124.568359375} \ No newline at end of file diff --git a/wandb/run-20260530_233437-fmmkx1a5/logs/debug-core.log b/wandb/run-20260530_233437-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..86bbbf8dc1809ccdd62a03891932f565d8183d16 --- /dev/null +++ b/wandb/run-20260530_233437-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-05-30T23:34:37.382417646+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpegcin8c4/port-3941550.txt","pid":3941550,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T23:34:37.383060104+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3941550} +{"time":"2026-05-30T23:34:37.383051621+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3941550-3942080-1657705318/socket","Net":"unix"}} +{"time":"2026-05-30T23:34:37.563078292+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T23:34:37.571164969+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T23:34:38.072654455+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T23:34:43.40609556+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rhy5otdvij68"} +{"time":"2026-05-30T23:53:03.26667659+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rhy5otdvij68"} +{"time":"2026-05-30T23:53:03.462142595+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rhy5otdvij68"} +{"time":"2026-05-30T23:53:03.464616525+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T23:53:03.465555637+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T23:53:04.132757107+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T23:53:04.132799262+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T23:53:04.132815029+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T23:53:04.132829063+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T23:53:04.132934455+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3941550-3942080-1657705318/socket","Net":"unix"}} +{"time":"2026-05-30T23:53:04.132976178+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T23:53:04.13300227+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T23:53:04.13302737+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260530_233437-fmmkx1a5/logs/debug-internal.log b/wandb/run-20260530_233437-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..aa651fa60f1f564d1a9a8806ca76dac94b240d0b --- /dev/null +++ b/wandb/run-20260530_233437-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,58 @@ +{"time":"2026-05-30T23:34:37.571383297+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T23:34:38.072095197+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T23:34:38.072329749+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T23:34:38.072407861+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T23:34:38.07263411+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T23:34:38.072672877+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T23:34:38.072680852+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T23:34:38.07695054+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T23:34:43.347016515+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T153443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=caf02b62cd5b67214ac9cb0821de216e38c56f330c0ff40852fdf4fab4d616ba"} +{"time":"2026-05-30T23:34:43.347173541+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B461CC6D2E5B4417e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_233437-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T153443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=caf02b62cd5b67214ac9cb0821de216e38c56f330c0ff40852fdf4fab4d616ba, Size: 2665}"} +{"time":"2026-05-30T23:34:43.35021857+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T153443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=878fff3033da323b3bc1b6aff3681358b3eebaaa95d35105f1b8d0ef6ceb32cf"} +{"time":"2026-05-30T23:34:43.350323305+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B461CC6D4EA34517e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_233437-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T153443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=878fff3033da323b3bc1b6aff3681358b3eebaaa95d35105f1b8d0ef6ceb32cf, Size: 2982}"} +{"time":"2026-05-30T23:34:43.359592273+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T153443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4bfb03c0c765fa68b5a55c86e4a3558269f5b023df859d8a1482bfce39d5145f"} +{"time":"2026-05-30T23:34:43.359686448+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff.patchnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff.patch18B461CC6CC0E83229678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_233437-fmmkx1a5/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T153443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4bfb03c0c765fa68b5a55c86e4a3558269f5b023df859d8a1482bfce39d5145f, Size: 14311}"} +{"time":"2026-05-30T23:34:43.359749281+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/train/rl_lavida.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T153443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=33ccb0bc0f1b2581206f388ea0ec56d6d5702eadd47554749c86e8d71f8d3185"} +{"time":"2026-05-30T23:34:43.359802824+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/train/rl_lavida.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/train/rl_lavida.py18B461CC6BAF35474b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_233437-fmmkx1a5/files/code/train/rl_lavida.py, Name: code/train/rl_lavida.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/train/rl_lavida.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T153443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=33ccb0bc0f1b2581206f388ea0ec56d6d5702eadd47554749c86e8d71f8d3185, Size: 32927}"} +{"time":"2026-05-30T23:34:43.360895283+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T153443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e29819181d72578e466a3d98646d89c5e954f1bd6ab57ecd56892bb2de1f434b"} +{"time":"2026-05-30T23:34:43.361067281+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18B461CC6BCBD5F94b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_233437-fmmkx1a5/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T153443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e29819181d72578e466a3d98646d89c5e954f1bd6ab57ecd56892bb2de1f434b, Size: 14311}"} +{"time":"2026-05-30T23:39:19.129136154+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":1,"current":39} +{"time":"2026-05-30T23:39:44.244451518+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":2,"current":39} +{"time":"2026-05-30T23:40:05.808335697+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":3,"current":39} +{"time":"2026-05-30T23:40:25.702705811+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":4,"current":39} +{"time":"2026-05-30T23:41:00.631894953+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":5,"current":39} +{"time":"2026-05-30T23:41:32.663304148+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":6,"current":39} +{"time":"2026-05-30T23:42:05.902924448+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":7,"current":39} +{"time":"2026-05-30T23:42:39.183483368+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":8,"current":39} +{"time":"2026-05-30T23:43:03.565994917+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":9,"current":39} +{"time":"2026-05-30T23:43:32.387427624+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":10,"current":39} +{"time":"2026-05-30T23:44:05.383331237+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":11,"current":39} +{"time":"2026-05-30T23:44:38.422863178+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":12,"current":39} +{"time":"2026-05-30T23:45:01.520006417+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":13,"current":39} +{"time":"2026-05-30T23:45:23.177395074+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":14,"current":39} +{"time":"2026-05-30T23:45:59.910880285+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":15,"current":39} +{"time":"2026-05-30T23:46:40.0807291+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":16,"current":39} +{"time":"2026-05-30T23:47:06.293051959+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":17,"current":39} +{"time":"2026-05-30T23:47:33.889398599+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":18,"current":39} +{"time":"2026-05-30T23:47:56.448811887+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":19,"current":39} +{"time":"2026-05-30T23:48:23.383912703+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":20,"current":39} +{"time":"2026-05-30T23:48:45.681835406+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":21,"current":39} +{"time":"2026-05-30T23:49:10.217289116+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":22,"current":39} +{"time":"2026-05-30T23:49:44.87279244+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":23,"current":39} +{"time":"2026-05-30T23:50:18.501600615+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":24,"current":39} +{"time":"2026-05-30T23:50:43.55116139+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":25,"current":39} +{"time":"2026-05-30T23:51:08.689325679+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":26,"current":39} +{"time":"2026-05-30T23:51:44.337047533+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":27,"current":39} +{"time":"2026-05-30T23:52:23.28666574+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":28,"current":39} +{"time":"2026-05-30T23:53:03.416702548+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T155303Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=09d8a6f5a83cc95d17e5fffe86dfa50be84c16f76335c48360011617b1421714"} +{"time":"2026-05-30T23:53:03.416808757+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B462CC8DEFBCD829678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_233437-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T155303Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=09d8a6f5a83cc95d17e5fffe86dfa50be84c16f76335c48360011617b1421714, Size: 7965}"} +{"time":"2026-05-30T23:53:03.417563853+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T155303Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e125b15e744b0837292e82dd9c854a2a07ccc3099b7787a5e9d24fb96e071b35"} +{"time":"2026-05-30T23:53:03.417681869+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B462CC8DF4DB8F29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_233437-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T155303Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e125b15e744b0837292e82dd9c854a2a07ccc3099b7787a5e9d24fb96e071b35, Size: 344}"} +{"time":"2026-05-30T23:53:03.418208085+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T155303Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ade55f79f4d76eca267d95031270543163225b84150b35794a9f35bdc087caba"} +{"time":"2026-05-30T23:53:03.418316486+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B462CC8E883E1C17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260530_233437-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T155303Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ade55f79f4d76eca267d95031270543163225b84150b35794a9f35bdc087caba, Size: 10119}"} +{"time":"2026-05-30T23:53:03.419272818+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T23:53:03.45976372+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T23:53:03.464637712+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T23:53:03.464655532+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T23:53:03.464732466+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T23:53:03.464741529+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/wandb/run-20260530_233437-fmmkx1a5/logs/debug.log b/wandb/run-20260530_233437-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c23a4c18169ac673531b996f1c4f8fc7c1508da0 --- /dev/null +++ b/wandb/run-20260530_233437-fmmkx1a5/logs/debug.log @@ -0,0 +1,26 @@ +2026-05-30 23:34:37,317 INFO MainThread:3941550 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 23:34:37,317 INFO MainThread:3941550 [wandb_setup.py:_flush():81] Configure stats pid to 3941550 +2026-05-30 23:34:37,317 INFO MainThread:3941550 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 23:34:37,318 INFO MainThread:3941550 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260530_233437-fmmkx1a5/logs/debug.log +2026-05-30 23:34:37,318 INFO MainThread:3941550 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260530_233437-fmmkx1a5/logs/debug-internal.log +2026-05-30 23:34:37,318 INFO MainThread:3941550 [wandb_init.py:init():844] calling init triggers +2026-05-30 23:34:37,318 INFO MainThread:3941550 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lavida.py'}} +2026-05-30 23:34:37,318 INFO MainThread:3941550 [wandb_init.py:init():892] starting backend +2026-05-30 23:34:37,563 INFO MainThread:3941550 [wandb_init.py:init():895] sending inform_init request +2026-05-30 23:34:37,567 INFO MainThread:3941550 [wandb_init.py:init():903] backend started and connected +2026-05-30 23:34:37,568 INFO MainThread:3941550 [wandb_init.py:init():973] updated telemetry +2026-05-30 23:34:37,574 INFO MainThread:3941550 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 23:34:38,274 INFO MainThread:3941550 [wandb_init.py:init():1037] run resumed +2026-05-30 23:34:38,278 INFO MainThread:3941550 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 23:34:38,401 INFO MainThread:3941550 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 23:34:38,402 INFO MainThread:3941550 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 23:34:38,402 INFO MainThread:3941550 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 23:34:38,402 INFO MainThread:3941550 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 23:34:38,406 INFO MainThread:3941550 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 23:34:38,407 INFO MainThread:3941550 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 26, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lavida_video_reversal/logs', 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lavida_video_reversal.yaml'} +2026-05-30 23:53:03,264 INFO MainThread:3941550 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 23:53:03,265 INFO MainThread:3941550 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 23:53:03,266 INFO MainThread:3941550 [wandb_run.py:_restore():2472] restore +2026-05-30 23:53:03,266 INFO MainThread:3941550 [wandb_run.py:_restore():2478] restore done +2026-05-30 23:53:03,463 INFO MainThread:3941550 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260601_091630-uynqqmyc/files/code/train/rl_llava_next_video.py b/wandb/run-20260601_091630-uynqqmyc/files/code/train/rl_llava_next_video.py new file mode 100644 index 0000000000000000000000000000000000000000..96a8ff1176bd3ff7dcf09cff8cad9583ae261198 --- /dev/null +++ b/wandb/run-20260601_091630-uynqqmyc/files/code/train/rl_llava_next_video.py @@ -0,0 +1,197 @@ +import copy +import os +import sys + +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import DataLoader, Dataset +import wandb +from accelerate import Accelerator + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LLAVA_NEXT_ROOT = os.path.join(REPO_ROOT, "LLaVA-NeXT") +sys.path.insert(0, LLAVA_NEXT_ROOT) +sys.path.insert(1, REPO_ROOT) + +from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX +from llava.conversation import conv_templates +from llava.mm_utils import tokenizer_image_token +from llava.model.builder import load_pretrained_model +from llava.utils import disable_torch_init, process_video_with_decord + +from train.llava_next_video.common import ( + build_video_mcq_prompt, + ensure_dir, + get_config, + load_json, + resolve_media_abs_path, +) +from train.utils import ensure_wandb_run_id, flatten_omega_conf + + +class RLDataset(Dataset): + def __init__(self, samples): + self.samples = samples + + def __len__(self): + return len(self.samples) + + def __getitem__(self, idx): + return self.samples[idx] + + +def build_prompt_text(sample: dict) -> str: + return build_video_mcq_prompt(sample) + + +def tokenize_example(tokenizer, conv_mode: str, prompt_text: str, response_text: str): + prompt_conv = copy.deepcopy(conv_templates[conv_mode]) + prompt_conv.append_message(prompt_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + prompt_conv.append_message(prompt_conv.roles[1], None) + prompt_ids = tokenizer_image_token(prompt_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + full_conv = copy.deepcopy(conv_templates[conv_mode]) + full_conv.append_message(full_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + full_conv.append_message(full_conv.roles[1], response_text) + full_ids = tokenizer_image_token(full_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + labels = full_ids.clone() + labels[: prompt_ids.numel()] = -100 + return full_ids, labels + + +def load_video_inputs(sample: dict, image_processor, model_cfg, device): + video_path = resolve_media_abs_path(sample, None, media_type="video") + video, _, _, _ = process_video_with_decord(video_path, model_cfg) + pixel_values = image_processor.preprocess(video, return_tensors="pt")["pixel_values"].to(device=device, dtype=torch.float16) + return [pixel_values], [video[0].size], ["video"] + + +def compute_token_logps(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: + shifted_logits = logits[:, :-1, :] + shifted_labels = labels[:, 1:] + valid_mask = shifted_labels.ne(-100) + gather_labels = shifted_labels.masked_fill(~valid_mask, 0) + token_logps = F.log_softmax(shifted_logits, dim=-1).gather(-1, gather_labels.unsqueeze(-1)).squeeze(-1) + return token_logps[valid_mask] + + +if __name__ == "__main__": + config = get_config() + disable_torch_init() + + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + ) + + existing_run_id = ensure_wandb_run_id(config) + if accelerator.is_main_process: + wandb.init( + project=config.experiment.project, + id=existing_run_id, + resume=config.wandb.resume, + config={k: v for k, v in flatten_omega_conf(config, resolve=True)}, + name=config.experiment.project, + reinit=True, + ) + + current_epoch = int(config.experiment.current_epoch) + model_path = config.model.pretrained_model if current_epoch == 1 else os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ref_model_path = config.model.reference_model or config.model.pretrained_model + train_samples = load_json(os.path.join(config.experiment.project, "temp_data", config.dataset.optimization_data_file)) + overwrite_config = dict(config.model.get("load_overwrite_config", {})) + ref_device = torch.device("cuda:1") if torch.cuda.is_available() and torch.cuda.device_count() > 1 else accelerator.device + + tokenizer, model, image_processor, _ = load_pretrained_model( + model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + _, ref_model, _, _ = load_pretrained_model( + ref_model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + ref_model.to(device=ref_device, dtype=torch.float16) + ref_model.eval() + for p in ref_model.parameters(): + p.requires_grad_(False) + + optimizer = AdamW(model.parameters(), lr=float(config.optimizer.learning_rate), weight_decay=float(config.optimizer.weight_decay)) + dataloader = DataLoader(RLDataset(train_samples), batch_size=1, shuffle=True) + + model, optimizer, dataloader = accelerator.prepare(model, optimizer, dataloader) + model.train() + + global_step = 0 + for _ in range(int(config.training.num_train_epochs)): + for batch in dataloader: + sample = {k: v[0] if isinstance(v, list) else v for k, v in batch.items()} + prompt_text = build_prompt_text(sample) + input_ids, labels = tokenize_example(tokenizer, config.model.conv_mode, prompt_text, sample["response"]) + input_ids = input_ids.unsqueeze(0).to(accelerator.device) + labels = labels.unsqueeze(0).to(accelerator.device) + attention_mask = input_ids.ne(tokenizer.pad_token_id if tokenizer.pad_token_id is not None else 0).long() + images, image_sizes, modalities = load_video_inputs(sample, image_processor, model.config, accelerator.device) + ref_input_ids = input_ids.to(ref_device) + ref_attention_mask = attention_mask.to(ref_device) + ref_images = [img.to(ref_device) for img in images] + ref_labels = labels.to(ref_device) + + with accelerator.accumulate(model): + outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + images=images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + with torch.no_grad(): + ref_outputs = ref_model( + input_ids=ref_input_ids, + attention_mask=ref_attention_mask, + images=ref_images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + + policy_logps = compute_token_logps(outputs.logits, labels) + ref_logps = compute_token_logps(ref_outputs.logits, ref_labels).to(policy_logps.device) + advantage = torch.tensor(float(sample["reward"]), device=accelerator.device, dtype=policy_logps.dtype) + pg_loss = -(advantage * policy_logps.mean()) + kl_loss = torch.clamp((policy_logps - ref_logps).mean(), min=0.0) + loss = pg_loss + float(config.training.kl_beta) * kl_loss + + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(model.parameters(), float(config.training.max_grad_norm)) + optimizer.step() + optimizer.zero_grad() + + global_step += 1 + if accelerator.is_main_process and global_step % int(config.training.logging_steps) == 0: + wandb.log( + { + "train/loss": float(loss.detach().cpu()), + "train/pg_loss": float(pg_loss.detach().cpu()), + "train/kl_loss": float(kl_loss.detach().cpu()), + "train/advantage": float(advantage.detach().cpu()), + "train/step": global_step, + } + ) + + accelerator.wait_for_everyone() + if accelerator.is_main_process: + out_dir = os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ensure_dir(out_dir) + accelerator.unwrap_model(model).save_pretrained(out_dir, safe_serialization=True) + tokenizer.save_pretrained(out_dir) + wandb.finish() diff --git a/wandb/run-20260601_091630-uynqqmyc/files/config.yaml b/wandb/run-20260601_091630-uynqqmyc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02f4d10b86f8b90adbf8e8db6be74da594528404 --- /dev/null +++ b/wandb/run-20260601_091630-uynqqmyc/files/config.yaml @@ -0,0 +1,206 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_llava_next_video.py + e: + ppeeppqh92r9kjjltoyq37va0vt073dz: + args: + - config=configs/rl_llava_next_video_reversal.yaml + - experiment.current_epoch=1 + - training.num_train_epochs=1 + - training.logging_steps=1 + codePath: train/rl_llava_next_video.py + codePathLocal: train/rl_llava_next_video.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "626324844544" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/llava-next-video/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_llava_next_video.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-06-01T01:16:30.420352Z" + writerId: ppeeppqh92r9kjjltoyq37va0vt073dz + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + "2": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + "3": + - 13 + - 14 + - 16 + - 41 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.40.0.dev0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_llava_next_video_reversal.yaml +dataset.optimization_data_file: + value: rl_data.json +dataset.train_json: + value: /data4/runqing/dLLM-RL/data/video_reversal_rl_train.json +evaluation.eval_json: + value: /data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json +evaluation.max_new_tokens: + value: 64 +evaluation.num_response_per_task: + value: 2 +evaluation.num_task_per_step: + value: 64 +evaluation.temperature: + value: 0.1 +experiment.current_epoch: + value: 1 +experiment.function: + value: train +experiment.project: + value: rl_llava_next_video_reversal +experiment.total_step: + value: 30 +model.conv_mode: + value: vicuna_v1 +model.load_overwrite_config.add_time_instruction: + value: true +model.load_overwrite_config.force_sample: + value: true +model.load_overwrite_config.frames_upbound: + value: 32 +model.load_overwrite_config.max_sequence_length: + value: 8192 +model.load_overwrite_config.mm_newline_position: + value: no_token +model.load_overwrite_config.mm_spatial_pool_mode: + value: average +model.load_overwrite_config.mm_spatial_pool_stride: + value: 2 +model.load_overwrite_config.mm_vision_tower: + value: /data4/runqing/hf/clip-vit-large-patch14-336 +model.load_overwrite_config.rope_scaling.factor: + value: 2 +model.load_overwrite_config.rope_scaling.type: + value: linear +model.load_overwrite_config.tokenizer_model_max_length: + value: 8192 +model.load_overwrite_config.video_fps: + value: 1 +model.model_base: + value: null +model.model_name: + value: llava_llama +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +model.reference_model: + value: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +optimizer.learning_rate: + value: 1e-06 +optimizer.weight_decay: + value: 0 +reward.direction_consistency_bonus: + value: 0.3 +reward.max_direction_gap: + value: 0.2 +rollout.max_new_tokens: + value: 64 +rollout.num_response_per_task: + value: 4 +rollout.num_task_per_step: + value: 128 +rollout.temperature: + value: 0.8 +rollout.top_p: + value: 0.95 +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/llava-next-video +training.gradient_accumulation_steps: + value: 2 +training.kl_beta: + value: 0.02 +training.logging_steps: + value: 1 +training.max_grad_norm: + value: 1 +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: uynqqmyc diff --git a/wandb/run-20260601_091630-uynqqmyc/files/diff.patch b/wandb/run-20260601_091630-uynqqmyc/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260601_091630-uynqqmyc/files/diff.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260601_091630-uynqqmyc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260601_091630-uynqqmyc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260601_091630-uynqqmyc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260601_091630-uynqqmyc/files/output.log b/wandb/run-20260601_091630-uynqqmyc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..53ffb5985ae44f1c6c86575a3b7d7c4136b2bbca --- /dev/null +++ b/wandb/run-20260601_091630-uynqqmyc/files/output.log @@ -0,0 +1,879 @@ +Loaded LLaVA model: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +You are using a model of type llava to instantiate a model of type llava_llama. This is not supported for all configurations of models and can yield errors. +Overwriting config with {'mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'mm_spatial_pool_stride': 2, 'mm_spatial_pool_mode': 'average', 'mm_newline_position': 'no_token', 'max_sequence_length': 8192, 'tokenizer_model_max_length': 8192, 'rope_scaling': {'factor': 2.0, 'type': 'linear'}, 'video_fps': 1, 'frames_upbound': 32, 'force_sample': True, 'add_time_instruction': True} +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +The checkpoint seems to contain `vision_tower` weights: `mm_tunable_parts` contains `mm_vision_tower`. +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.embeddings.class_embedding: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.embeddings.patch_embedding.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.embeddings.position_embedding.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.pre_layrnorm.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.pre_layrnorm.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.post_layernorm.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.post_layernorm.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +Loading checkpoint shards: 100%|██████████| 3/3 [00:02<00:00, 1.50it/s] +Model Class: LlavaLlamaForCausalLM +Loaded LLaVA model: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +You are using a model of type llava to instantiate a model of type llava_llama. This is not supported for all configurations of models and can yield errors. +Overwriting config with {'mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'mm_spatial_pool_stride': 2, 'mm_spatial_pool_mode': 'average', 'mm_newline_position': 'no_token', 'max_sequence_length': 8192, 'tokenizer_model_max_length': 8192, 'rope_scaling': {'factor': 2.0, 'type': 'linear'}, 'video_fps': 1, 'frames_upbound': 32, 'force_sample': True, 'add_time_instruction': True} +The checkpoint seems to contain `vision_tower` weights: `mm_tunable_parts` contains `mm_vision_tower`. +Loading checkpoint shards: 100%|██████████| 3/3 [00:01<00:00, 1.51it/s] +Model Class: LlavaLlamaForCausalLM +Traceback (most recent call last): + File "/data4/runqing/dLLM-RL/train/rl_llava_next_video.py", line 148, in + outputs = model( + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/accelerate/utils/operations.py", line 823, in forward + return model_forward(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/accelerate/utils/operations.py", line 811, in __call__ + return convert_to_fp32(self.model_forward(*args, **kwargs)) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast + return func(*args, **kwargs) + File "/data4/runqing/dLLM-RL/LLaVA-NeXT/llava/model/language_model/llava_llama.py", line 109, in forward + return super().forward( + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 1189, in forward + outputs = self.model( + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 1016, in forward + layer_outputs = decoder_layer( + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 739, in forward + hidden_states, self_attn_weights, present_key_value = self.self_attn( + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 436, in forward + key_states = self.k_proj(hidden_states) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/linear.py", line 125, in forward + return F.linear(input, self.weight, self.bias) +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 38.00 MiB. GPU 0 has a total capacity of 79.19 GiB of which 38.75 MiB is free. Including non-PyTorch memory, this process has 79.14 GiB memory in use. Of the allocated memory 73.48 GiB is allocated by PyTorch, and 4.93 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +Traceback (most recent call last): + File "/data4/runqing/dLLM-RL/train/rl_llava_next_video.py", line 148, in + outputs = model( + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/accelerate/utils/operations.py", line 823, in forward + return model_forward(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/accelerate/utils/operations.py", line 811, in __call__ + return convert_to_fp32(self.model_forward(*args, **kwargs)) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast + return func(*args, **kwargs) + File "/data4/runqing/dLLM-RL/LLaVA-NeXT/llava/model/language_model/llava_llama.py", line 109, in forward + return super().forward( + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 1189, in forward + outputs = self.model( + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 1016, in forward + layer_outputs = decoder_layer( + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 739, in forward + hidden_states, self_attn_weights, present_key_value = self.self_attn( + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 436, in forward + key_states = self.k_proj(hidden_states) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/linear.py", line 125, in forward + return F.linear(input, self.weight, self.bias) +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 38.00 MiB. GPU 0 has a total capacity of 79.19 GiB of which 38.75 MiB is free. Including non-PyTorch memory, this process has 79.14 GiB memory in use. Of the allocated memory 73.48 GiB is allocated by PyTorch, and 4.93 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) diff --git a/wandb/run-20260601_091630-uynqqmyc/files/requirements.txt b/wandb/run-20260601_091630-uynqqmyc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..75f723721d959cc9661b3ed252c3052267b34822 --- /dev/null +++ b/wandb/run-20260601_091630-uynqqmyc/files/requirements.txt @@ -0,0 +1,167 @@ +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +tokenizers==0.15.2 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +markdown-it-py==4.2.0 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +accelerate==1.13.0 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +six==1.17.0 +Pygments==2.19.2 +open_clip_torch==3.3.0 +transformers==4.40.0.dev0 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +mdurl==0.1.2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +rich==15.0.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +setuptools==80.10.2 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +peft==0.4.0 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +aiosignal==1.4.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +shtab==1.8.0 +opencv-python-headless==4.9.0.80 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +docstring_parser==0.18.0 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +tyro==0.8.3 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +ftfy==6.3.1 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +timm==1.0.27 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260601_091630-uynqqmyc/files/wandb-metadata.json b/wandb/run-20260601_091630-uynqqmyc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6b078d2f3e9a64e88c4f611a4c330ba8dc2d1802 --- /dev/null +++ b/wandb/run-20260601_091630-uynqqmyc/files/wandb-metadata.json @@ -0,0 +1,95 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-01T01:16:30.420352Z", + "args": [ + "config=configs/rl_llava_next_video_reversal.yaml", + "experiment.current_epoch=1", + "training.num_train_epochs=1", + "training.logging_steps=1" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_llava_next_video.py", + "codePath": "train/rl_llava_next_video.py", + "codePathLocal": "train/rl_llava_next_video.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/llava-next-video/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "626324844544" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "ppeeppqh92r9kjjltoyq37va0vt073dz" +} \ No newline at end of file diff --git a/wandb/run-20260601_091630-uynqqmyc/files/wandb-summary.json b/wandb/run-20260601_091630-uynqqmyc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9d62022ac4f47b9312647760f1dce5978253ca0d --- /dev/null +++ b/wandb/run-20260601_091630-uynqqmyc/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":17},"_step":1,"_runtime":17,"train/kl_loss":0,"train/step":2,"_timestamp":1.780276607880524e+09,"train/advantage":0,"train/loss":0,"train/pg_loss":0} \ No newline at end of file diff --git a/wandb/run-20260601_091630-uynqqmyc/logs/debug-core.log b/wandb/run-20260601_091630-uynqqmyc/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..dea9121f6a3908883af4ebe004eba246bedc91c1 --- /dev/null +++ b/wandb/run-20260601_091630-uynqqmyc/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-06-01T09:16:30.483288826+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpg48sapey/port-1597246.txt","pid":1597246,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-01T09:16:30.484189619+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1597246} +{"time":"2026-06-01T09:16:30.484118989+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1597246-1597363-3526337835/socket","Net":"unix"}} +{"time":"2026-06-01T09:16:30.667666219+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-01T09:16:30.676862672+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"uynqqmyc","id":"1(@)"} +{"time":"2026-06-01T09:16:31.18169394+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"uynqqmyc","id":"1(@)"} +{"time":"2026-06-01T09:16:36.531753471+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"hs9x4lxzay77"} +{"time":"2026-06-01T09:16:48.642889824+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-01T09:16:48.64301818+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-01T09:16:48.64301506+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-01T09:16:48.643174404+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-01T09:16:48.643152174+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1597246-1597363-3526337835/socket","Net":"unix"}} +{"time":"2026-06-01T09:16:48.784635764+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-01T09:16:48.784686579+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-01T09:16:48.784717922+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260601_091630-uynqqmyc/logs/debug-internal.log b/wandb/run-20260601_091630-uynqqmyc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..cc85390f269e6d9bdd4587200e0c6a2e1bbdf195 --- /dev/null +++ b/wandb/run-20260601_091630-uynqqmyc/logs/debug-internal.log @@ -0,0 +1,31 @@ +{"time":"2026-06-01T09:16:30.677064102+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-01T09:16:31.181233288+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-01T09:16:31.181391867+08:00","level":"INFO","msg":"stream: created new stream","id":"uynqqmyc"} +{"time":"2026-06-01T09:16:31.181476612+08:00","level":"INFO","msg":"handler: started","stream_id":"uynqqmyc"} +{"time":"2026-06-01T09:16:31.181671625+08:00","level":"INFO","msg":"stream: started","id":"uynqqmyc"} +{"time":"2026-06-01T09:16:31.181693695+08:00","level":"INFO","msg":"writer: started","stream_id":"uynqqmyc"} +{"time":"2026-06-01T09:16:31.181707411+08:00","level":"INFO","msg":"sender: started","stream_id":"uynqqmyc"} +{"time":"2026-06-01T09:16:31.186865083+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-01T09:16:32.066067312+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":52} +{"time":"2026-06-01T09:16:32.241508752+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":357} +{"time":"2026-06-01T09:16:36.488178683+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011636Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b41d5420dbf56313e894a33faf52380c40e1c4368abe0bddfecb901f955b7a04"} +{"time":"2026-06-01T09:16:36.488374633+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011636Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ca5ce644f76e128d903ff5c0cedbb67f597ada302e3266206093db007e01b902"} +{"time":"2026-06-01T09:16:36.488599102+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/requirements.txtnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/requirements.txt18B4D021D6F0569317e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_091630-uynqqmyc/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011636Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ca5ce644f76e128d903ff5c0cedbb67f597ada302e3266206093db007e01b902, Size: 3167}"} +{"time":"2026-06-01T09:16:36.488611319+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/wandb-metadata.json18B4D021D7311CDF4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_091630-uynqqmyc/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011636Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b41d5420dbf56313e894a33faf52380c40e1c4368abe0bddfecb901f955b7a04, Size: 2741}"} +{"time":"2026-06-01T09:16:36.490255985+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011636Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=59bf34275b83194859b07987940d8a4aadfcac8dbfb10afd8781698acf9743f0"} +{"time":"2026-06-01T09:16:36.490434168+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/code/train/rl_llava_next_video.pynetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/code/train/rl_llava_next_video.py18B4D021D65918E329678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_091630-uynqqmyc/files/code/train/rl_llava_next_video.py, Name: code/train/rl_llava_next_video.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011636Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=59bf34275b83194859b07987940d8a4aadfcac8dbfb10afd8781698acf9743f0, Size: 8539}"} +{"time":"2026-06-01T09:16:36.4938316+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011636Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7a7dbf4d612054ae9a18bc6117c6cc1c4c0c0b68968f3737fd16855cd1de2d4f"} +{"time":"2026-06-01T09:16:36.493918311+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/diff.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/diff.patch18B4D021D73F972E4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_091630-uynqqmyc/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011636Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7a7dbf4d612054ae9a18bc6117c6cc1c4c0c0b68968f3737fd16855cd1de2d4f, Size: 14311}"} +{"time":"2026-06-01T09:16:36.497868354+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011636Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=73a5d12868dae8ebfaa9192de0ec9407aa13882bb5fdfa2305a3b76c423551b0"} +{"time":"2026-06-01T09:16:36.497966049+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18B4D021D745AD424b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_091630-uynqqmyc/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011636Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=73a5d12868dae8ebfaa9192de0ec9407aa13882bb5fdfa2305a3b76c423551b0, Size: 14311}"} +{"time":"2026-06-01T09:16:48.64304174+08:00","level":"INFO","msg":"stream: closing","id":"uynqqmyc"} +{"time":"2026-06-01T09:16:48.708946157+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011648Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=97a862de585d37ad617291370effa5ea9bc0580fa1a8c984afd95e6a7acf8fd5"} +{"time":"2026-06-01T09:16:48.709082258+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/wandb-summary.json18B4D024AFA9B6B74b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_091630-uynqqmyc/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011648Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=97a862de585d37ad617291370effa5ea9bc0580fa1a8c984afd95e6a7acf8fd5, Size: 170}"} +{"time":"2026-06-01T09:16:48.72039872+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011648Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=fd9b83c6493b511690983b09b95369319881a3b683869ac58225aa808bd0db33"} +{"time":"2026-06-01T09:16:48.720454682+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/config.yamlnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/config.yaml18B4D024B00CB1E417e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_091630-uynqqmyc/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011648Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=fd9b83c6493b511690983b09b95369319881a3b683869ac58225aa808bd0db33, Size: 6760}"} +{"time":"2026-06-01T09:16:48.739711515+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011648Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=df54507570b71d8c07870450a34ce47c68bcf7186b036076b427107a336367e0"} +{"time":"2026-06-01T09:16:48.73984104+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/output.lognetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/output.log18B4D024AEC0F2F629678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_091630-uynqqmyc/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/uynqqmyc/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T011648Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=df54507570b71d8c07870450a34ce47c68bcf7186b036076b427107a336367e0, Size: 189477}"} +{"time":"2026-06-01T09:16:48.740761503+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-01T09:16:48.783896304+08:00","level":"INFO","msg":"handler: closed","stream_id":"uynqqmyc"} +{"time":"2026-06-01T09:16:48.78408553+08:00","level":"INFO","msg":"sender: closed","stream_id":"uynqqmyc"} +{"time":"2026-06-01T09:16:48.784113909+08:00","level":"INFO","msg":"stream: closed","id":"uynqqmyc"} diff --git a/wandb/run-20260601_091630-uynqqmyc/logs/debug.log b/wandb/run-20260601_091630-uynqqmyc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1b6d6a2c9326262c51c181dd859e533706c215d5 --- /dev/null +++ b/wandb/run-20260601_091630-uynqqmyc/logs/debug.log @@ -0,0 +1,21 @@ +2026-06-01 09:16:30,422 INFO MainThread:1597246 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-01 09:16:30,422 INFO MainThread:1597246 [wandb_setup.py:_flush():81] Configure stats pid to 1597246 +2026-06-01 09:16:30,422 INFO MainThread:1597246 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-01 09:16:30,422 INFO MainThread:1597246 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260601_091630-uynqqmyc/logs/debug.log +2026-06-01 09:16:30,422 INFO MainThread:1597246 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260601_091630-uynqqmyc/logs/debug-internal.log +2026-06-01 09:16:30,422 INFO MainThread:1597246 [wandb_init.py:init():844] calling init triggers +2026-06-01 09:16:30,422 INFO MainThread:1597246 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'uynqqmyc', 'system.env_name': '/data8/runqing/miniconda/envs/llava-next-video', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_llava_next_video_reversal', 'experiment.function': 'train', 'experiment.current_epoch': 1, 'experiment.total_step': 30, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.reference_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_name': 'llava_llama', 'model.model_base': None, 'model.conv_mode': 'vicuna_v1', 'model.load_overwrite_config.mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'model.load_overwrite_config.mm_spatial_pool_stride': 2, 'model.load_overwrite_config.mm_spatial_pool_mode': 'average', 'model.load_overwrite_config.mm_newline_position': 'no_token', 'model.load_overwrite_config.max_sequence_length': 8192, 'model.load_overwrite_config.tokenizer_model_max_length': 8192, 'model.load_overwrite_config.rope_scaling.factor': 2.0, 'model.load_overwrite_config.rope_scaling.type': 'linear', 'model.load_overwrite_config.video_fps': 1, 'model.load_overwrite_config.frames_upbound': 32, 'model.load_overwrite_config.force_sample': True, 'model.load_overwrite_config.add_time_instruction': True, 'dataset.train_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_train.json', 'dataset.optimization_data_file': 'rl_data.json', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 4, 'rollout.temperature': 0.8, 'rollout.top_p': 0.95, 'rollout.max_new_tokens': 64, 'reward.max_direction_gap': 0.2, 'reward.direction_consistency_bonus': 0.3, 'training.num_train_epochs': 1, 'training.gradient_accumulation_steps': 2, 'training.mixed_precision': 'bf16', 'training.max_grad_norm': 1.0, 'training.kl_beta': 0.02, 'training.logging_steps': 1, 'optimizer.learning_rate': 1e-06, 'optimizer.weight_decay': 0.0, 'evaluation.eval_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json', 'evaluation.num_task_per_step': 64, 'evaluation.num_response_per_task': 2, 'evaluation.max_new_tokens': 64, 'evaluation.temperature': 0.1, 'config': 'configs/rl_llava_next_video_reversal.yaml', '_wandb': {'code_path': 'code/train/rl_llava_next_video.py'}} +2026-06-01 09:16:30,422 INFO MainThread:1597246 [wandb_init.py:init():892] starting backend +2026-06-01 09:16:30,668 INFO MainThread:1597246 [wandb_init.py:init():895] sending inform_init request +2026-06-01 09:16:30,672 INFO MainThread:1597246 [wandb_init.py:init():903] backend started and connected +2026-06-01 09:16:30,674 INFO MainThread:1597246 [wandb_init.py:init():973] updated telemetry +2026-06-01 09:16:30,677 INFO MainThread:1597246 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-01 09:16:31,414 INFO MainThread:1597246 [wandb_init.py:init():1042] starting run threads in backend +2026-06-01 09:16:31,528 INFO MainThread:1597246 [wandb_run.py:_console_start():2524] atexit reg +2026-06-01 09:16:31,528 INFO MainThread:1597246 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-01 09:16:31,528 INFO MainThread:1597246 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-01 09:16:31,528 INFO MainThread:1597246 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-01 09:16:31,532 INFO MainThread:1597246 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-01 09:16:48,642 INFO wandb-AsyncioManager-main:1597246 [service_client.py:_forward_responses():134] Reached EOF. +2026-06-01 09:16:48,643 INFO wandb-AsyncioManager-main:1597246 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/wandb/run-20260601_103407-7qsz55yy/files/code/train/rl_llava_next_video.py b/wandb/run-20260601_103407-7qsz55yy/files/code/train/rl_llava_next_video.py new file mode 100644 index 0000000000000000000000000000000000000000..ef9c138cb736c609b6fb4f8e79f0b0c684681aa1 --- /dev/null +++ b/wandb/run-20260601_103407-7qsz55yy/files/code/train/rl_llava_next_video.py @@ -0,0 +1,325 @@ +import copy +import os +import sys +from typing import Iterable, List + +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import DataLoader, Dataset +import wandb +from accelerate import Accelerator +from omegaconf import OmegaConf + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LLAVA_NEXT_ROOT = os.path.join(REPO_ROOT, "LLaVA-NeXT") +sys.path.insert(0, LLAVA_NEXT_ROOT) +sys.path.insert(1, REPO_ROOT) + +from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX +from llava.conversation import conv_templates +from llava.mm_utils import tokenizer_image_token +from llava.model.builder import load_pretrained_model +from llava.utils import disable_torch_init, process_video_with_decord + +from train.llava_next_video.common import ( + build_video_mcq_prompt, + ensure_dir, + get_config, + load_json, + resolve_media_abs_path, +) +from train.utils import ensure_wandb_run_id, flatten_omega_conf + + +class RLDataset(Dataset): + def __init__(self, samples): + self.samples = samples + + def __len__(self): + return len(self.samples) + + def __getitem__(self, idx): + return self.samples[idx] + + +def build_prompt_text(sample: dict) -> str: + return build_video_mcq_prompt(sample) + + +def tokenize_example(tokenizer, conv_mode: str, prompt_text: str, response_text: str): + prompt_conv = copy.deepcopy(conv_templates[conv_mode]) + prompt_conv.append_message(prompt_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + prompt_conv.append_message(prompt_conv.roles[1], None) + prompt_ids = tokenizer_image_token(prompt_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + full_conv = copy.deepcopy(conv_templates[conv_mode]) + full_conv.append_message(full_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + full_conv.append_message(full_conv.roles[1], response_text) + full_ids = tokenizer_image_token(full_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + labels = full_ids.clone() + labels[: prompt_ids.numel()] = -100 + return full_ids, labels + + +def load_video_inputs(sample: dict, image_processor, model_cfg, device): + video_path = resolve_media_abs_path(sample, None, media_type="video") + video, _, _, _ = process_video_with_decord(video_path, model_cfg) + pixel_values = image_processor.preprocess(video, return_tensors="pt")["pixel_values"].to(device=device, dtype=torch.float16) + return [pixel_values], [video[0].size], ["video"] + + +def compute_token_logps(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: + shifted_logits = logits[:, :-1, :] + shifted_labels = labels[:, 1:] + valid_mask = shifted_labels.ne(-100) + gather_labels = shifted_labels.masked_fill(~valid_mask, 0) + token_logps = F.log_softmax(shifted_logits, dim=-1).gather(-1, gather_labels.unsqueeze(-1)).squeeze(-1) + token_logps = token_logps[valid_mask] + return token_logps[torch.isfinite(token_logps)] + + +def safe_mean(values: torch.Tensor, *, device: torch.device, dtype: torch.dtype) -> torch.Tensor: + if values.numel() == 0: + return torch.zeros((), device=device, dtype=dtype) + return values.mean() + + +def snapshot_named_parameters(model, names: Iterable[str]) -> dict[str, torch.Tensor]: + state = {} + target_names = set(names) + for name, param in model.named_parameters(): + if name in target_names: + state[name] = param.detach().cpu().clone() + return state + + +def restore_named_parameters(model, state: dict[str, torch.Tensor]) -> None: + for name, param in model.named_parameters(): + if name in state: + param.data.copy_(state[name].to(device=param.device, dtype=param.dtype)) + + +def gradients_are_finite(params: Iterable[torch.nn.Parameter]) -> bool: + for param in params: + if param.grad is not None and not torch.isfinite(param.grad).all(): + return False + return True + + +def parameters_are_finite(params: Iterable[torch.nn.Parameter]) -> bool: + return all(torch.isfinite(param).all() for param in params) + + +def _merge_overwrite_config(config) -> dict: + overwrite_config = OmegaConf.to_container(config.model.get("load_overwrite_config", {}), resolve=True) or {} + training_overrides = OmegaConf.to_container(config.training.get("load_overwrite_config", {}), resolve=True) or {} + overwrite_config.update(training_overrides) + return overwrite_config + + +def _set_trainable_parameters(model, trainable_parts: Iterable[str]) -> List[torch.nn.Parameter]: + part_to_patterns = { + "lm_head": ["lm_head"], + "embed_tokens": ["embed_tokens"], + "mm_projector": ["mm_projector"], + "vision_resampler": ["vision_resampler"], + "vision_tower": ["vision_tower"], + } + enabled_parts = list(trainable_parts) + enabled_patterns = [] + for part in enabled_parts: + if part == "all": + for param in model.parameters(): + param.requires_grad_(True) + return list(model.parameters()) + enabled_patterns.extend(part_to_patterns.get(part, [part])) + + trainable_params: List[torch.nn.Parameter] = [] + seen_param_ids = set() + for name, param in model.named_parameters(): + should_train = any(pattern in name for pattern in enabled_patterns) + param.requires_grad_(should_train) + if should_train and id(param) not in seen_param_ids: + trainable_params.append(param) + seen_param_ids.add(id(param)) + return trainable_params + + +def _summarize_trainable_parameters(model) -> tuple[int, int, List[str]]: + total_params = 0 + trainable_params = 0 + trainable_names = [] + for name, param in model.named_parameters(): + count = param.numel() + total_params += count + if param.requires_grad: + trainable_params += count + trainable_names.append(name) + return trainable_params, total_params, trainable_names + + +if __name__ == "__main__": + config = get_config() + disable_torch_init() + + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + ) + + existing_run_id = ensure_wandb_run_id(config) + if accelerator.is_main_process: + wandb.init( + project=config.experiment.project, + id=existing_run_id, + resume=config.wandb.resume, + config={k: v for k, v in flatten_omega_conf(config, resolve=True)}, + name=config.experiment.project, + reinit=True, + ) + + current_epoch = int(config.experiment.current_epoch) + model_path = config.model.pretrained_model if current_epoch == 1 else os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ref_model_path = config.model.reference_model or config.model.pretrained_model + train_samples = load_json(os.path.join(config.experiment.project, "temp_data", config.dataset.optimization_data_file)) + overwrite_config = _merge_overwrite_config(config) + ref_device = torch.device("cuda:1") if torch.cuda.is_available() and torch.cuda.device_count() > 1 else accelerator.device + + tokenizer, model, image_processor, _ = load_pretrained_model( + model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + _, ref_model, _, _ = load_pretrained_model( + ref_model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + ref_model.to(device=ref_device, dtype=torch.float16) + ref_model.eval() + for p in ref_model.parameters(): + p.requires_grad_(False) + + trainable_parts = list(config.training.get("trainable_parts", ["lm_head"])) + trainable_params = _set_trainable_parameters(model, trainable_parts) + if not trainable_params: + raise ValueError(f"No trainable parameters matched training.trainable_parts={trainable_parts}") + + trainable_count, total_count, trainable_names = _summarize_trainable_parameters(model) + if accelerator.is_main_process: + print( + f"RL trainable parts: {trainable_parts}; " + f"trainable params={trainable_count:,}/{total_count:,} " + f"({100.0 * trainable_count / max(total_count, 1):.4f}%)" + ) + for name in trainable_names[:20]: + print(f" trainable: {name}") + if len(trainable_names) > 20: + print(f" ... and {len(trainable_names) - 20} more") + + optimizer = AdamW(trainable_params, lr=float(config.optimizer.learning_rate), weight_decay=float(config.optimizer.weight_decay)) + dataloader = DataLoader(RLDataset(train_samples), batch_size=1, shuffle=True) + + model, optimizer, dataloader = accelerator.prepare(model, optimizer, dataloader) + last_good_state = snapshot_named_parameters(accelerator.unwrap_model(model), trainable_names) + model.train() + + global_step = 0 + skipped_steps = 0 + for _ in range(int(config.training.num_train_epochs)): + for batch in dataloader: + sample = {k: v[0] if isinstance(v, list) else v for k, v in batch.items()} + prompt_text = build_prompt_text(sample) + input_ids, labels = tokenize_example(tokenizer, config.model.conv_mode, prompt_text, sample["response"]) + input_ids = input_ids.unsqueeze(0).to(accelerator.device) + labels = labels.unsqueeze(0).to(accelerator.device) + attention_mask = input_ids.ne(tokenizer.pad_token_id if tokenizer.pad_token_id is not None else 0).long() + images, image_sizes, modalities = load_video_inputs(sample, image_processor, model.config, accelerator.device) + ref_input_ids = input_ids.to(ref_device) + ref_attention_mask = attention_mask.to(ref_device) + ref_images = [img.to(ref_device) for img in images] + ref_labels = labels.to(ref_device) + + with accelerator.accumulate(model): + outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + images=images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + with torch.no_grad(): + ref_outputs = ref_model( + input_ids=ref_input_ids, + attention_mask=ref_attention_mask, + images=ref_images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + + policy_logps = compute_token_logps(outputs.logits, labels) + ref_logps = compute_token_logps(ref_outputs.logits, ref_labels).to(policy_logps.device) + shared_tokens = min(policy_logps.numel(), ref_logps.numel()) + if shared_tokens == 0: + skipped_steps += 1 + optimizer.zero_grad() + continue + policy_logps = policy_logps[:shared_tokens] + ref_logps = ref_logps[:shared_tokens] + advantage = torch.tensor(float(sample["reward"]), device=accelerator.device, dtype=policy_logps.dtype) + pg_loss = -(advantage * safe_mean(policy_logps, device=accelerator.device, dtype=policy_logps.dtype)) + kl_loss = torch.clamp( + safe_mean(policy_logps - ref_logps, device=accelerator.device, dtype=policy_logps.dtype), + min=0.0, + ) + loss = pg_loss + float(config.training.kl_beta) * kl_loss + if not torch.isfinite(loss): + skipped_steps += 1 + optimizer.zero_grad() + continue + + accelerator.backward(loss) + if not gradients_are_finite(trainable_params): + skipped_steps += 1 + optimizer.zero_grad() + restore_named_parameters(accelerator.unwrap_model(model), last_good_state) + continue + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, float(config.training.max_grad_norm)) + optimizer.step() + if not parameters_are_finite(trainable_params): + skipped_steps += 1 + restore_named_parameters(accelerator.unwrap_model(model), last_good_state) + optimizer.zero_grad() + continue + last_good_state = snapshot_named_parameters(accelerator.unwrap_model(model), trainable_names) + optimizer.zero_grad() + + global_step += 1 + if accelerator.is_main_process and global_step % int(config.training.logging_steps) == 0: + wandb.log( + { + "train/loss": float(loss.detach().cpu()), + "train/pg_loss": float(pg_loss.detach().cpu()), + "train/kl_loss": float(kl_loss.detach().cpu()), + "train/advantage": float(advantage.detach().cpu()), + "train/skipped_steps": skipped_steps, + "train/step": global_step, + } + ) + + accelerator.wait_for_everyone() + if accelerator.is_main_process: + out_dir = os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ensure_dir(out_dir) + accelerator.unwrap_model(model).save_pretrained(out_dir, safe_serialization=True) + tokenizer.save_pretrained(out_dir) + wandb.finish() diff --git a/wandb/run-20260601_103407-7qsz55yy/files/config.yaml b/wandb/run-20260601_103407-7qsz55yy/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09f4bf98cb840739642b7d63bedab72606759b53 --- /dev/null +++ b/wandb/run-20260601_103407-7qsz55yy/files/config.yaml @@ -0,0 +1,215 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_llava_next_video.py + e: + djcvs25yuvzcqlza5e6jcctexs1p18iq: + args: + - config=configs/rl_llava_next_video_reversal.yaml + - experiment.current_epoch=1 + - training.num_train_epochs=1 + - training.logging_steps=50 + codePath: train/rl_llava_next_video.py + codePathLocal: train/rl_llava_next_video.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "626044514304" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/llava-next-video/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_llava_next_video.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-06-01T02:34:07.260164Z" + writerId: djcvs25yuvzcqlza5e6jcctexs1p18iq + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + "2": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + "3": + - 2 + - 13 + - 14 + - 16 + - 41 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.40.0.dev0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_llava_next_video_reversal.yaml +dataset.optimization_data_file: + value: rl_data.json +dataset.train_json: + value: /data4/runqing/dLLM-RL/data/video_reversal_rl_train.json +evaluation.eval_json: + value: /data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json +evaluation.max_new_tokens: + value: 64 +evaluation.num_response_per_task: + value: 2 +evaluation.num_task_per_step: + value: 64 +evaluation.temperature: + value: 0.1 +experiment.current_epoch: + value: 1 +experiment.function: + value: train +experiment.project: + value: rl_llava_next_video_reversal +experiment.total_step: + value: 30 +model.conv_mode: + value: vicuna_v1 +model.load_overwrite_config.add_time_instruction: + value: true +model.load_overwrite_config.force_sample: + value: true +model.load_overwrite_config.frames_upbound: + value: 32 +model.load_overwrite_config.max_sequence_length: + value: 8192 +model.load_overwrite_config.mm_newline_position: + value: no_token +model.load_overwrite_config.mm_spatial_pool_mode: + value: average +model.load_overwrite_config.mm_spatial_pool_stride: + value: 2 +model.load_overwrite_config.mm_vision_tower: + value: /data4/runqing/hf/clip-vit-large-patch14-336 +model.load_overwrite_config.rope_scaling.factor: + value: 2 +model.load_overwrite_config.rope_scaling.type: + value: linear +model.load_overwrite_config.tokenizer_model_max_length: + value: 8192 +model.load_overwrite_config.video_fps: + value: 1 +model.model_base: + value: null +model.model_name: + value: llava_llama +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +model.reference_model: + value: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +optimizer.learning_rate: + value: 1e-06 +optimizer.weight_decay: + value: 0 +reward.direction_consistency_bonus: + value: 0.3 +reward.max_direction_gap: + value: 0.2 +rollout.max_new_tokens: + value: 64 +rollout.num_response_per_task: + value: 4 +rollout.num_task_per_step: + value: 128 +rollout.temperature: + value: 0.8 +rollout.top_p: + value: 0.95 +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/llava-next-video +training.gradient_accumulation_steps: + value: 2 +training.kl_beta: + value: 0.02 +training.load_overwrite_config.frames_upbound: + value: 8 +training.load_overwrite_config.max_sequence_length: + value: 2048 +training.load_overwrite_config.tokenizer_model_max_length: + value: 2048 +training.logging_steps: + value: 50 +training.max_grad_norm: + value: 1 +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.trainable_parts.0: + value: lm_head +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: 7qsz55yy diff --git a/wandb/run-20260601_103407-7qsz55yy/files/diff.patch b/wandb/run-20260601_103407-7qsz55yy/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260601_103407-7qsz55yy/files/diff.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260601_103407-7qsz55yy/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260601_103407-7qsz55yy/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260601_103407-7qsz55yy/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260601_103407-7qsz55yy/files/output.log b/wandb/run-20260601_103407-7qsz55yy/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..dda7c2a62555d098011d2f97368b0bb6d4d3d55e --- /dev/null +++ b/wandb/run-20260601_103407-7qsz55yy/files/output.log @@ -0,0 +1,797 @@ +Loaded LLaVA model: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +You are using a model of type llava to instantiate a model of type llava_llama. This is not supported for all configurations of models and can yield errors. +Overwriting config with {'mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'mm_spatial_pool_stride': 2, 'mm_spatial_pool_mode': 'average', 'mm_newline_position': 'no_token', 'max_sequence_length': 2048, 'tokenizer_model_max_length': 2048, 'rope_scaling': {'factor': 2.0, 'type': 'linear'}, 'video_fps': 1, 'frames_upbound': 8, 'force_sample': True, 'add_time_instruction': True} +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +The checkpoint seems to contain `vision_tower` weights: `mm_tunable_parts` contains `mm_vision_tower`. +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.embeddings.class_embedding: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.embeddings.patch_embedding.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.embeddings.position_embedding.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.pre_layrnorm.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.pre_layrnorm.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.0.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.1.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.2.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.3.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.4.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.5.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.6.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.7.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.8.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.9.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.10.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.11.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.12.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.13.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.14.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.15.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.16.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.17.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.18.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.19.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.20.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.21.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.22.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.k_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.k_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.v_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.v_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.q_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.q_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.out_proj.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.self_attn.out_proj.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.layer_norm1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.layer_norm1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.mlp.fc1.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.mlp.fc1.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.mlp.fc2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.mlp.fc2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.layer_norm2.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.encoder.layers.23.layer_norm2.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.post_layernorm.weight: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +/data8/runqing/miniconda/envs/llava-next-video/lib/python3.10/site-packages/torch/nn/modules/module.py:2397: UserWarning: for vision_model.post_layernorm.bias: copying from a non-meta parameter in the checkpoint to a meta parameter in the current model, which is a no-op. (Did you mean to pass `assign=True` to assign items in the state dictionary to their corresponding key in the module instead of copying them in place?) + warnings.warn( +Loading checkpoint shards: 100%|██████████| 3/3 [00:01<00:00, 1.79it/s] +Model Class: LlavaLlamaForCausalLM +Loaded LLaVA model: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +You are using a model of type llava to instantiate a model of type llava_llama. This is not supported for all configurations of models and can yield errors. +Overwriting config with {'mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'mm_spatial_pool_stride': 2, 'mm_spatial_pool_mode': 'average', 'mm_newline_position': 'no_token', 'max_sequence_length': 2048, 'tokenizer_model_max_length': 2048, 'rope_scaling': {'factor': 2.0, 'type': 'linear'}, 'video_fps': 1, 'frames_upbound': 8, 'force_sample': True, 'add_time_instruction': True} +The checkpoint seems to contain `vision_tower` weights: `mm_tunable_parts` contains `mm_vision_tower`. +Loading checkpoint shards: 100%|██████████| 3/3 [00:01<00:00, 1.77it/s] +Model Class: LlavaLlamaForCausalLM +RL trainable parts: ['lm_head']; trainable params=131,076,096/7,062,915,072 (1.8558%) + trainable: lm_head.weight diff --git a/wandb/run-20260601_103407-7qsz55yy/files/requirements.txt b/wandb/run-20260601_103407-7qsz55yy/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..75f723721d959cc9661b3ed252c3052267b34822 --- /dev/null +++ b/wandb/run-20260601_103407-7qsz55yy/files/requirements.txt @@ -0,0 +1,167 @@ +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +tokenizers==0.15.2 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +markdown-it-py==4.2.0 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +accelerate==1.13.0 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +six==1.17.0 +Pygments==2.19.2 +open_clip_torch==3.3.0 +transformers==4.40.0.dev0 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +mdurl==0.1.2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +rich==15.0.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +setuptools==80.10.2 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +peft==0.4.0 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +aiosignal==1.4.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +shtab==1.8.0 +opencv-python-headless==4.9.0.80 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +docstring_parser==0.18.0 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +tyro==0.8.3 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +ftfy==6.3.1 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +timm==1.0.27 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260601_103407-7qsz55yy/files/wandb-metadata.json b/wandb/run-20260601_103407-7qsz55yy/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..01319db9662fe33184219761eed871764ac233e1 --- /dev/null +++ b/wandb/run-20260601_103407-7qsz55yy/files/wandb-metadata.json @@ -0,0 +1,95 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-01T02:34:07.260164Z", + "args": [ + "config=configs/rl_llava_next_video_reversal.yaml", + "experiment.current_epoch=1", + "training.num_train_epochs=1", + "training.logging_steps=50" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_llava_next_video.py", + "codePath": "train/rl_llava_next_video.py", + "codePathLocal": "train/rl_llava_next_video.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/llava-next-video/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "626044514304" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "djcvs25yuvzcqlza5e6jcctexs1p18iq" +} \ No newline at end of file diff --git a/wandb/run-20260601_103407-7qsz55yy/files/wandb-summary.json b/wandb/run-20260601_103407-7qsz55yy/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..079d9369ed9624e5426152ed93a0f5e592949514 --- /dev/null +++ b/wandb/run-20260601_103407-7qsz55yy/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/skipped_steps":249,"_step":4,"train/advantage":0,"train/pg_loss":0,"train/kl_loss":0.0035479962825775146,"train/step":250,"_runtime":327,"train/loss":7.09599262336269e-05,"_wandb":{"runtime":327},"_timestamp":1.7802815312796028e+09} \ No newline at end of file diff --git a/wandb/run-20260601_103407-7qsz55yy/logs/debug-core.log b/wandb/run-20260601_103407-7qsz55yy/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..e88409325c1912bf9ed19f68f5e84e880959cc34 --- /dev/null +++ b/wandb/run-20260601_103407-7qsz55yy/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-06-01T10:34:07.326738487+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_mrg08iy/port-1641154.txt","pid":1641154,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-01T10:34:07.32738888+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1641154} +{"time":"2026-06-01T10:34:07.327386943+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1641154-1641499-2685067064/socket","Net":"unix"}} +{"time":"2026-06-01T10:34:07.506789729+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-01T10:34:07.516619732+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"7qsz55yy","id":"1(@)"} +{"time":"2026-06-01T10:34:07.815687561+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"7qsz55yy","id":"1(@)"} +{"time":"2026-06-01T10:34:13.175410797+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"i8611nb1coub"} +{"time":"2026-06-01T10:39:35.572897985+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"i8611nb1coub"} +{"time":"2026-06-01T10:39:35.793770204+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"i8611nb1coub"} +{"time":"2026-06-01T10:39:35.794692683+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"7qsz55yy","id":"1(@)"} +{"time":"2026-06-01T10:39:35.795966756+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"7qsz55yy","id":"1(@)"} +{"time":"2026-06-01T10:39:35.810296823+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-01T10:39:35.810318038+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-01T10:39:35.810325876+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-01T10:39:35.810329396+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-01T10:39:35.810385632+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1641154-1641499-2685067064/socket","Net":"unix"}} +{"time":"2026-06-01T10:39:35.810412508+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-01T10:39:35.810430128+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-01T10:39:35.810447523+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260601_103407-7qsz55yy/logs/debug-internal.log b/wandb/run-20260601_103407-7qsz55yy/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..929c88ec819cebd8030e0a6f7b5bb28813de75d7 --- /dev/null +++ b/wandb/run-20260601_103407-7qsz55yy/logs/debug-internal.log @@ -0,0 +1,32 @@ +{"time":"2026-06-01T10:34:07.516789633+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-01T10:34:07.81454049+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-01T10:34:07.814810783+08:00","level":"INFO","msg":"stream: created new stream","id":"7qsz55yy"} +{"time":"2026-06-01T10:34:07.814895708+08:00","level":"INFO","msg":"handler: started","stream_id":"7qsz55yy"} +{"time":"2026-06-01T10:34:07.815564912+08:00","level":"INFO","msg":"stream: started","id":"7qsz55yy"} +{"time":"2026-06-01T10:34:07.816701006+08:00","level":"INFO","msg":"sender: started","stream_id":"7qsz55yy"} +{"time":"2026-06-01T10:34:07.816700981+08:00","level":"INFO","msg":"writer: started","stream_id":"7qsz55yy"} +{"time":"2026-06-01T10:34:07.819496319+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-01T10:34:08.605966385+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":52} +{"time":"2026-06-01T10:34:08.775034579+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":357} +{"time":"2026-06-01T10:34:13.129958874+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023413Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f33a644014fdef0413766102eb2d890e94432e84963061fcb8914754a72ed951"} +{"time":"2026-06-01T10:34:13.130121954+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/wandb-metadata.json18B4D45E0C9E2DB94b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_103407-7qsz55yy/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023413Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f33a644014fdef0413766102eb2d890e94432e84963061fcb8914754a72ed951, Size: 2742}"} +{"time":"2026-06-01T10:34:13.134312683+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023413Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0f025f274d37f439f783286e67bacf33b215585cf37de50587aea7de9ce6f755"} +{"time":"2026-06-01T10:34:13.134400688+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/requirements.txtnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/requirements.txt18B4D45E0BE2B6EC29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_103407-7qsz55yy/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023413Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0f025f274d37f439f783286e67bacf33b215585cf37de50587aea7de9ce6f755, Size: 3167}"} +{"time":"2026-06-01T10:34:13.141333442+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023413Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=21d801a2812fb1553c12e7edfab5241fb20c1f3466212d4949e2574f06e701f4"} +{"time":"2026-06-01T10:34:13.141432658+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/code/train/rl_llava_next_video.pynetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/code/train/rl_llava_next_video.py18B4D45E0C64142317e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_103407-7qsz55yy/files/code/train/rl_llava_next_video.py, Name: code/train/rl_llava_next_video.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023413Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=21d801a2812fb1553c12e7edfab5241fb20c1f3466212d4949e2574f06e701f4, Size: 13916}"} +{"time":"2026-06-01T10:34:13.142792229+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023413Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2188dd4220a7cc4ffccc213c73167ad04eda9a916fc260eb4ac939e8855dd21b"} +{"time":"2026-06-01T10:34:13.142890954+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18B4D45E0C7495C717e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_103407-7qsz55yy/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023413Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2188dd4220a7cc4ffccc213c73167ad04eda9a916fc260eb4ac939e8855dd21b, Size: 14311}"} +{"time":"2026-06-01T10:34:13.144663023+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023413Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=364a4f5067cef39f461f5d3fdaa573c713bdb29e633500f27b589d22ebd6142a"} +{"time":"2026-06-01T10:34:13.144848719+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/diff.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/diff.patch18B4D45E0C8B83C117e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_103407-7qsz55yy/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023413Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=364a4f5067cef39f461f5d3fdaa573c713bdb29e633500f27b589d22ebd6142a, Size: 14311}"} +{"time":"2026-06-01T10:39:35.717415002+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023935Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=395ae906a5a623b0676ff5ef80c44bf14276aa30d67c8565a692cacac89846ab"} +{"time":"2026-06-01T10:39:35.717533476+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/wandb-summary.json18B4D4A92754094D29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_103407-7qsz55yy/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023935Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=395ae906a5a623b0676ff5ef80c44bf14276aa30d67c8565a692cacac89846ab, Size: 240}"} +{"time":"2026-06-01T10:39:35.718704825+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023935Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b6d0c57fce8451489f2508f831698fd77c65b3978fd355db1938f5dc15975072"} +{"time":"2026-06-01T10:39:35.718848871+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/config.yamlnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/config.yaml18B4D4A92804D38417e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_103407-7qsz55yy/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023935Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b6d0c57fce8451489f2508f831698fd77c65b3978fd355db1938f5dc15975072, Size: 7032}"} +{"time":"2026-06-01T10:39:35.74970116+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023935Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ba5d310fd9db70523bb4ef54d82d5f19aba10f9a71844968e6f13ee91c857b84"} +{"time":"2026-06-01T10:39:35.749822441+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/output.lognetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/output.log18B4D4A927ECBAB117e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_103407-7qsz55yy/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/7qsz55yy/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T023935Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ba5d310fd9db70523bb4ef54d82d5f19aba10f9a71844968e6f13ee91c857b84, Size: 181155}"} +{"time":"2026-06-01T10:39:35.750619272+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-01T10:39:35.789542707+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-01T10:39:35.794715421+08:00","level":"INFO","msg":"stream: closing","id":"7qsz55yy"} +{"time":"2026-06-01T10:39:35.794730384+08:00","level":"INFO","msg":"handler: closed","stream_id":"7qsz55yy"} +{"time":"2026-06-01T10:39:35.794785746+08:00","level":"INFO","msg":"sender: closed","stream_id":"7qsz55yy"} +{"time":"2026-06-01T10:39:35.794800515+08:00","level":"INFO","msg":"stream: closed","id":"7qsz55yy"} diff --git a/wandb/run-20260601_103407-7qsz55yy/logs/debug.log b/wandb/run-20260601_103407-7qsz55yy/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..85951f88fb2c0c46a7ff873e742ece0198e318e1 --- /dev/null +++ b/wandb/run-20260601_103407-7qsz55yy/logs/debug.log @@ -0,0 +1,24 @@ +2026-06-01 10:34:07,263 INFO MainThread:1641154 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-01 10:34:07,263 INFO MainThread:1641154 [wandb_setup.py:_flush():81] Configure stats pid to 1641154 +2026-06-01 10:34:07,263 INFO MainThread:1641154 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-01 10:34:07,263 INFO MainThread:1641154 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260601_103407-7qsz55yy/logs/debug.log +2026-06-01 10:34:07,263 INFO MainThread:1641154 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260601_103407-7qsz55yy/logs/debug-internal.log +2026-06-01 10:34:07,263 INFO MainThread:1641154 [wandb_init.py:init():844] calling init triggers +2026-06-01 10:34:07,263 INFO MainThread:1641154 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': '7qsz55yy', 'system.env_name': '/data8/runqing/miniconda/envs/llava-next-video', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_llava_next_video_reversal', 'experiment.function': 'train', 'experiment.current_epoch': 1, 'experiment.total_step': 30, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.reference_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_name': 'llava_llama', 'model.model_base': None, 'model.conv_mode': 'vicuna_v1', 'model.load_overwrite_config.mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'model.load_overwrite_config.mm_spatial_pool_stride': 2, 'model.load_overwrite_config.mm_spatial_pool_mode': 'average', 'model.load_overwrite_config.mm_newline_position': 'no_token', 'model.load_overwrite_config.max_sequence_length': 8192, 'model.load_overwrite_config.tokenizer_model_max_length': 8192, 'model.load_overwrite_config.rope_scaling.factor': 2.0, 'model.load_overwrite_config.rope_scaling.type': 'linear', 'model.load_overwrite_config.video_fps': 1, 'model.load_overwrite_config.frames_upbound': 32, 'model.load_overwrite_config.force_sample': True, 'model.load_overwrite_config.add_time_instruction': True, 'dataset.train_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_train.json', 'dataset.optimization_data_file': 'rl_data.json', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 4, 'rollout.temperature': 0.8, 'rollout.top_p': 0.95, 'rollout.max_new_tokens': 64, 'reward.max_direction_gap': 0.2, 'reward.direction_consistency_bonus': 0.3, 'training.num_train_epochs': 1, 'training.gradient_accumulation_steps': 2, 'training.mixed_precision': 'bf16', 'training.max_grad_norm': 1.0, 'training.kl_beta': 0.02, 'training.logging_steps': 50, 'training.trainable_parts.0': 'lm_head', 'training.load_overwrite_config.max_sequence_length': 2048, 'training.load_overwrite_config.tokenizer_model_max_length': 2048, 'training.load_overwrite_config.frames_upbound': 8, 'optimizer.learning_rate': 1e-06, 'optimizer.weight_decay': 0.0, 'evaluation.eval_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json', 'evaluation.num_task_per_step': 64, 'evaluation.num_response_per_task': 2, 'evaluation.max_new_tokens': 64, 'evaluation.temperature': 0.1, 'config': 'configs/rl_llava_next_video_reversal.yaml', '_wandb': {'code_path': 'code/train/rl_llava_next_video.py'}} +2026-06-01 10:34:07,263 INFO MainThread:1641154 [wandb_init.py:init():892] starting backend +2026-06-01 10:34:07,507 INFO MainThread:1641154 [wandb_init.py:init():895] sending inform_init request +2026-06-01 10:34:07,512 INFO MainThread:1641154 [wandb_init.py:init():903] backend started and connected +2026-06-01 10:34:07,513 INFO MainThread:1641154 [wandb_init.py:init():973] updated telemetry +2026-06-01 10:34:07,623 INFO MainThread:1641154 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-01 10:34:08,059 INFO MainThread:1641154 [wandb_init.py:init():1042] starting run threads in backend +2026-06-01 10:34:08,172 INFO MainThread:1641154 [wandb_run.py:_console_start():2524] atexit reg +2026-06-01 10:34:08,172 INFO MainThread:1641154 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-01 10:34:08,172 INFO MainThread:1641154 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-01 10:34:08,172 INFO MainThread:1641154 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-01 10:34:08,176 INFO MainThread:1641154 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-01 10:39:35,571 INFO MainThread:1641154 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_llava_next_video_reversal/7qsz55yy +2026-06-01 10:39:35,572 INFO MainThread:1641154 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-01 10:39:35,572 INFO MainThread:1641154 [wandb_run.py:_restore():2472] restore +2026-06-01 10:39:35,572 INFO MainThread:1641154 [wandb_run.py:_restore():2478] restore done +2026-06-01 10:39:35,794 INFO MainThread:1641154 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260601_125307-9x12ylug/files/diff.patch b/wandb/run-20260601_125307-9x12ylug/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260601_125307-9x12ylug/files/diff.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260601_125307-9x12ylug/files/wandb-metadata.json b/wandb/run-20260601_125307-9x12ylug/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3ecd48e748cb872c161cd778b89767d24c475700 --- /dev/null +++ b/wandb/run-20260601_125307-9x12ylug/files/wandb-metadata.json @@ -0,0 +1,94 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-01T04:53:07.277485Z", + "args": [ + "config=configs/rl_llava_next_video_reversal.yaml", + "wandb.run_id=9x12ylug", + "experiment.current_epoch=5" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_llava_next_video.py", + "codePath": "train/rl_llava_next_video.py", + "codePathLocal": "train/rl_llava_next_video.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/llava-next-video/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "626051198976" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "9iv6m8fu2indssiyqsu0rq0ul79ddtgu" +} \ No newline at end of file diff --git a/wandb/run-20260601_125307-9x12ylug/files/wandb-summary.json b/wandb/run-20260601_125307-9x12ylug/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3cb0176b27868d0ae7671c8ff073190b8eb3a311 --- /dev/null +++ b/wandb/run-20260601_125307-9x12ylug/files/wandb-summary.json @@ -0,0 +1 @@ +{"_step":2559,"_runtime":2903,"train/loss":8.03286361694336,"_wandb":{"runtime":2903},"train/skipped_steps":511,"train/step":512,"train/pg_loss":8.03286361694336,"train/advantage":1,"train/kl_loss":0,"_timestamp":1.7802901280705595e+09} \ No newline at end of file diff --git a/wandb/run-20260601_125307-9x12ylug/logs/debug-core.log b/wandb/run-20260601_125307-9x12ylug/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..9be22f4d34b278f1da34c07b04e965f7466725ec --- /dev/null +++ b/wandb/run-20260601_125307-9x12ylug/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-06-01T12:53:07.345410787+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpqbx9ep1r/port-1958880.txt","pid":1958880,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-01T12:53:07.346128692+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1958880} +{"time":"2026-06-01T12:53:07.346105009+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1958880-1959137-870286764/socket","Net":"unix"}} +{"time":"2026-06-01T12:53:07.525908072+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-01T12:53:07.534706992+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T12:53:07.839654587+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T12:53:13.184984356+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"i74g46lf7zwq"} +{"time":"2026-06-01T13:02:45.702576823+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"i74g46lf7zwq"} +{"time":"2026-06-01T13:02:45.903053781+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"i74g46lf7zwq"} +{"time":"2026-06-01T13:02:45.903563948+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T13:02:45.90467427+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T13:02:45.91760251+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-01T13:02:45.91763165+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-01T13:02:45.917648352+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-01T13:02:45.917653202+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-01T13:02:45.917761893+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1958880-1959137-870286764/socket","Net":"unix"}} +{"time":"2026-06-01T13:02:45.917820248+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-01T13:02:45.917840538+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-01T13:02:45.917864982+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260601_125307-9x12ylug/logs/debug-internal.log b/wandb/run-20260601_125307-9x12ylug/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0e3157632021f81ad9aa05ad111f8d84f2e3a325 --- /dev/null +++ b/wandb/run-20260601_125307-9x12ylug/logs/debug-internal.log @@ -0,0 +1,30 @@ +{"time":"2026-06-01T12:53:07.534817146+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-01T12:53:07.836953027+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-01T12:53:07.837145031+08:00","level":"INFO","msg":"stream: created new stream","id":"9x12ylug"} +{"time":"2026-06-01T12:53:07.837222019+08:00","level":"INFO","msg":"handler: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T12:53:07.839570886+08:00","level":"INFO","msg":"stream: started","id":"9x12ylug"} +{"time":"2026-06-01T12:53:07.839626803+08:00","level":"INFO","msg":"sender: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T12:53:07.839629101+08:00","level":"INFO","msg":"writer: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T12:53:07.842890308+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-01T12:53:13.136760961+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T045313Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=27b894a8b43100988ea01bf2cba46f2fc451e3aed5ea2e3a7dff4f56d9fe2aed"} +{"time":"2026-06-01T12:53:13.136939849+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txtnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt18B4DBF3DB8A0FCE4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_125307-9x12ylug/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T045313Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=27b894a8b43100988ea01bf2cba46f2fc451e3aed5ea2e3a7dff4f56d9fe2aed, Size: 3167}"} +{"time":"2026-06-01T12:53:13.142978788+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T045313Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=bac36bec2d610459e43ed8f36b4f817ea9841e956df54835ebbcd4f2e923507c"} +{"time":"2026-06-01T12:53:13.143065553+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json18B4DBF3DADB061A29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_125307-9x12ylug/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T045313Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=bac36bec2d610459e43ed8f36b4f817ea9841e956df54835ebbcd4f2e923507c, Size: 2703}"} +{"time":"2026-06-01T12:53:13.143944182+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T045313Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=31551fe04fe9447396656fff61864ee01c50b6e5ffb19453de93cce21455189f"} +{"time":"2026-06-01T12:53:13.144054421+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.pynetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py18B4DBF3DA8F869829678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_125307-9x12ylug/files/code/train/rl_llava_next_video.py, Name: code/train/rl_llava_next_video.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T045313Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=31551fe04fe9447396656fff61864ee01c50b6e5ffb19453de93cce21455189f, Size: 15437}"} +{"time":"2026-06-01T12:53:13.144246462+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T045313Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9e72a81b3a65a032f762e262287e6ffbba57ac930b636b53977cd42b8dfc003f"} +{"time":"2026-06-01T12:53:13.145835323+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch18B4DBF3DB29C05417e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_125307-9x12ylug/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T045313Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9e72a81b3a65a032f762e262287e6ffbba57ac930b636b53977cd42b8dfc003f, Size: 14311}"} +{"time":"2026-06-01T12:53:13.148770612+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T045313Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=62cc0b703f9396aa33809c7091d44eafd009e96fc08bdc1a21a0135f5d357dc7"} +{"time":"2026-06-01T12:53:13.148836958+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18B4DBF3DAC2FBA029678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_125307-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T045313Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=62cc0b703f9396aa33809c7091d44eafd009e96fc08bdc1a21a0135f5d357dc7, Size: 14311}"} +{"time":"2026-06-01T13:02:45.838840525+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T050245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c3d8df82149fd547ad7c1db79fbf5a1a0b3ff8fbbdec27cb6cca66d5d8820dbf"} +{"time":"2026-06-01T13:02:45.838942147+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json18B4DC7932CB7FD917e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_125307-9x12ylug/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T050245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c3d8df82149fd547ad7c1db79fbf5a1a0b3ff8fbbdec27cb6cca66d5d8820dbf, Size: 236}"} +{"time":"2026-06-01T13:02:45.84053618+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T050245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=baffd01a7a91200191f380d2029f42907db4e5f3ee87d0f90766b213f1ac9aa9"} +{"time":"2026-06-01T13:02:45.840591502+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yamlnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml18B4DC79323FEFCD29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_125307-9x12ylug/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T050245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=baffd01a7a91200191f380d2029f42907db4e5f3ee87d0f90766b213f1ac9aa9, Size: 7147}"} +{"time":"2026-06-01T13:02:45.85283515+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T050245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=06a4264cdc02d8847add7220fd702948d2201bd658e8b4c8fc8ebe98a6e13ce8"} +{"time":"2026-06-01T13:02:45.852938883+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.lognetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log18B4DC79333261E34b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_125307-9x12ylug/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T050245Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=06a4264cdc02d8847add7220fd702948d2201bd658e8b4c8fc8ebe98a6e13ce8, Size: 48345}"} +{"time":"2026-06-01T13:02:45.85393916+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-01T13:02:45.898441844+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-01T13:02:45.90358046+08:00","level":"INFO","msg":"stream: closing","id":"9x12ylug"} +{"time":"2026-06-01T13:02:45.903593758+08:00","level":"INFO","msg":"handler: closed","stream_id":"9x12ylug"} +{"time":"2026-06-01T13:02:45.903650378+08:00","level":"INFO","msg":"sender: closed","stream_id":"9x12ylug"} +{"time":"2026-06-01T13:02:45.903664256+08:00","level":"INFO","msg":"stream: closed","id":"9x12ylug"} diff --git a/wandb/run-20260601_125307-9x12ylug/logs/debug.log b/wandb/run-20260601_125307-9x12ylug/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e20e5a1c4e10fa5c1faac7f31adda00a076551ca --- /dev/null +++ b/wandb/run-20260601_125307-9x12ylug/logs/debug.log @@ -0,0 +1,25 @@ +2026-06-01 12:53:07,280 INFO MainThread:1958880 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-01 12:53:07,281 INFO MainThread:1958880 [wandb_setup.py:_flush():81] Configure stats pid to 1958880 +2026-06-01 12:53:07,281 INFO MainThread:1958880 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-01 12:53:07,281 INFO MainThread:1958880 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260601_125307-9x12ylug/logs/debug.log +2026-06-01 12:53:07,281 INFO MainThread:1958880 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260601_125307-9x12ylug/logs/debug-internal.log +2026-06-01 12:53:07,281 INFO MainThread:1958880 [wandb_init.py:init():844] calling init triggers +2026-06-01 12:53:07,281 INFO MainThread:1958880 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': '9x12ylug', 'system.env_name': '/data8/runqing/miniconda/envs/llava-next-video', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_llava_next_video_reversal', 'experiment.function': 'train', 'experiment.current_epoch': 5, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.reference_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_name': 'llava_llama', 'model.model_base': None, 'model.conv_mode': 'vicuna_v1', 'model.load_overwrite_config.mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'model.load_overwrite_config.mm_spatial_pool_stride': 2, 'model.load_overwrite_config.mm_spatial_pool_mode': 'average', 'model.load_overwrite_config.mm_newline_position': 'no_token', 'model.load_overwrite_config.max_sequence_length': 8192, 'model.load_overwrite_config.tokenizer_model_max_length': 8192, 'model.load_overwrite_config.rope_scaling.factor': 2.0, 'model.load_overwrite_config.rope_scaling.type': 'linear', 'model.load_overwrite_config.video_fps': 1, 'model.load_overwrite_config.frames_upbound': 32, 'model.load_overwrite_config.force_sample': True, 'model.load_overwrite_config.add_time_instruction': True, 'dataset.train_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_train.json', 'dataset.optimization_data_file': 'rl_data.json', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.top_p': 0.95, 'rollout.max_new_tokens': 128, 'reward.max_direction_gap': 0.2, 'reward.direction_consistency_bonus': 0.3, 'training.num_train_epochs': 1, 'training.gradient_accumulation_steps': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.max_grad_norm': 1.0, 'training.kl_beta': 0.01, 'training.logging_steps': 1, 'training.trainable_parts.0': 'mm_projector', 'training.load_overwrite_config.max_sequence_length': 2048, 'training.load_overwrite_config.tokenizer_model_max_length': 2048, 'training.load_overwrite_config.frames_upbound': 8, 'optimizer.learning_rate': 1e-06, 'optimizer.weight_decay': 0.0, 'evaluation.eval_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json', 'evaluation.num_task_per_step': 64, 'evaluation.num_response_per_task': 3, 'evaluation.max_new_tokens': 128, 'evaluation.temperature': 0.1, 'config': 'configs/rl_llava_next_video_reversal.yaml', '_wandb': {'code_path': 'code/train/rl_llava_next_video.py'}} +2026-06-01 12:53:07,281 INFO MainThread:1958880 [wandb_init.py:init():892] starting backend +2026-06-01 12:53:07,526 INFO MainThread:1958880 [wandb_init.py:init():895] sending inform_init request +2026-06-01 12:53:07,531 INFO MainThread:1958880 [wandb_init.py:init():903] backend started and connected +2026-06-01 12:53:07,533 INFO MainThread:1958880 [wandb_init.py:init():973] updated telemetry +2026-06-01 12:53:07,538 INFO MainThread:1958880 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-01 12:53:08,061 INFO MainThread:1958880 [wandb_init.py:init():1037] run resumed +2026-06-01 12:53:08,065 INFO MainThread:1958880 [wandb_init.py:init():1042] starting run threads in backend +2026-06-01 12:53:08,180 INFO MainThread:1958880 [wandb_run.py:_console_start():2524] atexit reg +2026-06-01 12:53:08,180 INFO MainThread:1958880 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-01 12:53:08,180 INFO MainThread:1958880 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-01 12:53:08,181 INFO MainThread:1958880 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-01 12:53:08,185 INFO MainThread:1958880 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-01 13:02:45,701 INFO MainThread:1958880 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_llava_next_video_reversal/9x12ylug +2026-06-01 13:02:45,702 INFO MainThread:1958880 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-01 13:02:45,702 INFO MainThread:1958880 [wandb_run.py:_restore():2472] restore +2026-06-01 13:02:45,702 INFO MainThread:1958880 [wandb_run.py:_restore():2478] restore done +2026-06-01 13:02:45,902 INFO MainThread:1958880 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260601_132214-9x12ylug/files/code/train/rl_llava_next_video.py b/wandb/run-20260601_132214-9x12ylug/files/code/train/rl_llava_next_video.py new file mode 100644 index 0000000000000000000000000000000000000000..8199163c2bde56855ea46bec0831fe434956172b --- /dev/null +++ b/wandb/run-20260601_132214-9x12ylug/files/code/train/rl_llava_next_video.py @@ -0,0 +1,358 @@ +import copy +import os +import sys +from typing import Iterable, List +import warnings +import random + +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import DataLoader, Dataset +import wandb +from accelerate import Accelerator +from omegaconf import OmegaConf + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LLAVA_NEXT_ROOT = os.path.join(REPO_ROOT, "LLaVA-NeXT") +sys.path.insert(0, LLAVA_NEXT_ROOT) +sys.path.insert(1, REPO_ROOT) + +from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX +from llava.conversation import conv_templates +from llava.mm_utils import tokenizer_image_token +from llava.model.builder import load_pretrained_model +from llava.utils import disable_torch_init, process_video_with_decord + +from train.llava_next_video.common import ( + build_video_mcq_prompt, + ensure_dir, + get_config, + load_json, + resolve_media_abs_path, +) +from train.utils import ensure_wandb_run_id, flatten_omega_conf + +warnings.filterwarnings("ignore", message=".*copying from a non-meta parameter.*") +warnings.filterwarnings("ignore", message=".*Flash Attention 2.0 with a model not initialized on GPU.*") + + +class RLDataset(Dataset): + def __init__(self, samples): + self.samples = samples + + def __len__(self): + return len(self.samples) + + def __getitem__(self, idx): + return self.samples[idx] + + +def build_prompt_text(sample: dict) -> str: + return build_video_mcq_prompt(sample) + + +def tokenize_example(tokenizer, conv_mode: str, prompt_text: str, response_text: str): + prompt_conv = copy.deepcopy(conv_templates[conv_mode]) + prompt_conv.append_message(prompt_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + prompt_conv.append_message(prompt_conv.roles[1], None) + prompt_ids = tokenizer_image_token(prompt_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + full_conv = copy.deepcopy(conv_templates[conv_mode]) + full_conv.append_message(full_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + full_conv.append_message(full_conv.roles[1], response_text) + full_ids = tokenizer_image_token(full_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + labels = full_ids.clone() + labels[: prompt_ids.numel()] = -100 + return full_ids, labels + + +def load_video_inputs(sample: dict, image_processor, model_cfg, device): + video_path = resolve_media_abs_path(sample, None, media_type="video") + video, _, _, _ = process_video_with_decord(video_path, model_cfg) + pixel_values = image_processor.preprocess(video, return_tensors="pt")["pixel_values"].to(device=device, dtype=torch.float16) + return [pixel_values], [video[0].size], ["video"] + + +def compute_token_logps(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: + shifted_logits = logits[:, :-1, :] + shifted_labels = labels[:, 1:] + valid_mask = shifted_labels.ne(-100) + gather_labels = shifted_labels.masked_fill(~valid_mask, 0) + token_logps = F.log_softmax(shifted_logits, dim=-1).gather(-1, gather_labels.unsqueeze(-1)).squeeze(-1) + token_logps = token_logps[valid_mask] + return token_logps[torch.isfinite(token_logps)] + + +def safe_mean(values: torch.Tensor, *, device: torch.device, dtype: torch.dtype) -> torch.Tensor: + if values.numel() == 0: + return torch.zeros((), device=device, dtype=dtype) + return values.mean() + + +def snapshot_named_parameters(model, names: Iterable[str]) -> dict[str, torch.Tensor]: + state = {} + target_names = set(names) + for name, param in model.named_parameters(): + if name in target_names: + state[name] = param.detach().cpu().clone() + return state + + +def restore_named_parameters(model, state: dict[str, torch.Tensor]) -> None: + for name, param in model.named_parameters(): + if name in state: + param.data.copy_(state[name].to(device=param.device, dtype=param.dtype)) + + +def gradients_are_finite(params: Iterable[torch.nn.Parameter]) -> bool: + for param in params: + if param.grad is not None and not torch.isfinite(param.grad).all(): + return False + return True + + +def parameters_are_finite(params: Iterable[torch.nn.Parameter]) -> bool: + return all(torch.isfinite(param).all() for param in params) + + +def _merge_overwrite_config(config) -> dict: + overwrite_config = OmegaConf.to_container(config.model.get("load_overwrite_config", {}), resolve=True) or {} + training_overrides = OmegaConf.to_container(config.training.get("load_overwrite_config", {}), resolve=True) or {} + overwrite_config.update(training_overrides) + return overwrite_config + + +def _set_trainable_parameters(model, trainable_parts: Iterable[str]) -> List[torch.nn.Parameter]: + part_to_patterns = { + "lm_head": ["lm_head"], + "embed_tokens": ["embed_tokens"], + "mm_projector": ["mm_projector"], + "vision_resampler": ["vision_resampler"], + "vision_tower": ["vision_tower"], + } + enabled_parts = list(trainable_parts) + enabled_patterns = [] + for part in enabled_parts: + if part == "all": + for param in model.parameters(): + param.requires_grad_(True) + return list(model.parameters()) + enabled_patterns.extend(part_to_patterns.get(part, [part])) + + trainable_params: List[torch.nn.Parameter] = [] + seen_param_ids = set() + for name, param in model.named_parameters(): + should_train = any(pattern in name for pattern in enabled_patterns) + param.requires_grad_(should_train) + if should_train and id(param) not in seen_param_ids: + trainable_params.append(param) + seen_param_ids.add(id(param)) + return trainable_params + + +def _summarize_trainable_parameters(model) -> tuple[int, int, List[str]]: + total_params = 0 + trainable_params = 0 + trainable_names = [] + for name, param in model.named_parameters(): + count = param.numel() + total_params += count + if param.requires_grad: + trainable_params += count + trainable_names.append(name) + return trainable_params, total_params, trainable_names + + +def set_seed(seed: int) -> None: + random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + + +if __name__ == "__main__": + config = get_config() + disable_torch_init() + set_seed(int(config.training.get("seed", 10086))) + if hasattr(torch.backends, "cuda") and torch.cuda.is_available(): + torch.backends.cuda.matmul.allow_tf32 = bool(config.training.get("enable_tf32", True)) + if hasattr(torch.backends, "cudnn"): + torch.backends.cudnn.allow_tf32 = bool(config.training.get("enable_tf32", True)) + + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + ) + + existing_run_id = ensure_wandb_run_id(config) + if accelerator.is_main_process: + wandb.init( + project=config.experiment.project, + id=existing_run_id, + resume=config.wandb.resume, + config={k: v for k, v in flatten_omega_conf(config, resolve=True)}, + name=config.experiment.project, + reinit=True, + ) + + current_epoch = int(config.experiment.current_epoch) + model_path = config.model.pretrained_model if current_epoch == 1 else os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ref_model_path = config.model.reference_model or config.model.pretrained_model + train_samples = load_json(os.path.join(config.experiment.project, "temp_data", config.dataset.optimization_data_file)) + overwrite_config = _merge_overwrite_config(config) + ref_device = torch.device("cuda:1") if torch.cuda.is_available() and torch.cuda.device_count() > 1 else accelerator.device + if accelerator.is_main_process: + print( + f"[train] epoch={current_epoch} samples={len(train_samples)} " + f"grad_accum={int(config.training.gradient_accumulation_steps)}", + flush=True, + ) + + tokenizer, model, image_processor, _ = load_pretrained_model( + model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + _, ref_model, _, _ = load_pretrained_model( + ref_model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + ref_model.to(device=ref_device, dtype=torch.float16) + ref_model.eval() + for p in ref_model.parameters(): + p.requires_grad_(False) + + trainable_parts = list(config.training.get("trainable_parts", ["lm_head"])) + trainable_params = _set_trainable_parameters(model, trainable_parts) + if not trainable_params: + raise ValueError(f"No trainable parameters matched training.trainable_parts={trainable_parts}") + + trainable_count, total_count, trainable_names = _summarize_trainable_parameters(model) + if accelerator.is_main_process: + print( + f"RL trainable parts: {trainable_parts}; " + f"trainable params={trainable_count:,}/{total_count:,} " + f"({100.0 * trainable_count / max(total_count, 1):.4f}%)" + ) + for name in trainable_names[:20]: + print(f" trainable: {name}") + if len(trainable_names) > 20: + print(f" ... and {len(trainable_names) - 20} more") + + optimizer = AdamW(trainable_params, lr=float(config.optimizer.learning_rate), weight_decay=float(config.optimizer.weight_decay)) + dataloader = DataLoader(RLDataset(train_samples), batch_size=1, shuffle=True) + + model, optimizer, dataloader = accelerator.prepare(model, optimizer, dataloader) + last_good_state = snapshot_named_parameters(accelerator.unwrap_model(model), trainable_names) + model.train() + + global_step = 0 + skipped_steps = 0 + for _ in range(int(config.training.num_train_epochs)): + for batch in dataloader: + sample = {k: v[0] if isinstance(v, list) else v for k, v in batch.items()} + prompt_text = build_prompt_text(sample) + input_ids, labels = tokenize_example(tokenizer, config.model.conv_mode, prompt_text, sample["response"]) + input_ids = input_ids.unsqueeze(0).to(accelerator.device) + labels = labels.unsqueeze(0).to(accelerator.device) + attention_mask = input_ids.ne(tokenizer.pad_token_id if tokenizer.pad_token_id is not None else 0).long() + images, image_sizes, modalities = load_video_inputs(sample, image_processor, model.config, accelerator.device) + ref_input_ids = input_ids.to(ref_device) + ref_attention_mask = attention_mask.to(ref_device) + ref_images = [img.to(ref_device) for img in images] + ref_labels = labels.to(ref_device) + + with accelerator.accumulate(model): + outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + images=images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + with torch.no_grad(): + ref_outputs = ref_model( + input_ids=ref_input_ids, + attention_mask=ref_attention_mask, + images=ref_images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + + policy_logps = compute_token_logps(outputs.logits, labels) + ref_logps = compute_token_logps(ref_outputs.logits, ref_labels).to(policy_logps.device) + shared_tokens = min(policy_logps.numel(), ref_logps.numel()) + if shared_tokens == 0: + skipped_steps += 1 + optimizer.zero_grad() + continue + policy_logps = policy_logps[:shared_tokens] + ref_logps = ref_logps[:shared_tokens] + advantage = torch.tensor(float(sample["reward"]), device=accelerator.device, dtype=policy_logps.dtype) + pg_loss = -(advantage * safe_mean(policy_logps, device=accelerator.device, dtype=policy_logps.dtype)) + kl_loss = torch.clamp( + safe_mean(policy_logps - ref_logps, device=accelerator.device, dtype=policy_logps.dtype), + min=0.0, + ) + loss = pg_loss + float(config.training.kl_beta) * kl_loss + if not torch.isfinite(loss): + skipped_steps += 1 + optimizer.zero_grad() + continue + + accelerator.backward(loss) + if not gradients_are_finite(trainable_params): + skipped_steps += 1 + optimizer.zero_grad() + restore_named_parameters(accelerator.unwrap_model(model), last_good_state) + continue + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, float(config.training.max_grad_norm)) + optimizer.step() + if not parameters_are_finite(trainable_params): + skipped_steps += 1 + restore_named_parameters(accelerator.unwrap_model(model), last_good_state) + optimizer.zero_grad() + continue + last_good_state = snapshot_named_parameters(accelerator.unwrap_model(model), trainable_names) + optimizer.zero_grad() + + global_step += 1 + if accelerator.is_main_process and global_step % int(config.training.logging_steps) == 0: + print( + f"[train] epoch={current_epoch} step={global_step} " + f"loss={float(loss.detach().cpu()):.6f} " + f"pg={float(pg_loss.detach().cpu()):.6f} " + f"kl={float(kl_loss.detach().cpu()):.6f} " + f"reward={float(advantage.detach().cpu()):.6f} " + f"skipped={skipped_steps}", + flush=True, + ) + wandb.log( + { + "train/loss": float(loss.detach().cpu()), + "train/pg_loss": float(pg_loss.detach().cpu()), + "train/kl_loss": float(kl_loss.detach().cpu()), + "train/advantage": float(advantage.detach().cpu()), + "train/skipped_steps": skipped_steps, + "train/step": global_step, + } + ) + + accelerator.wait_for_everyone() + if accelerator.is_main_process: + out_dir = os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ensure_dir(out_dir) + accelerator.unwrap_model(model).save_pretrained(out_dir, safe_serialization=True) + tokenizer.save_pretrained(out_dir) + print(f"[train] epoch={current_epoch} checkpoint_saved={os.path.abspath(out_dir)}", flush=True) + wandb.finish() diff --git a/wandb/run-20260601_133700-9x12ylug/files/code/train/rl_llava_next_video.py b/wandb/run-20260601_133700-9x12ylug/files/code/train/rl_llava_next_video.py new file mode 100644 index 0000000000000000000000000000000000000000..8199163c2bde56855ea46bec0831fe434956172b --- /dev/null +++ b/wandb/run-20260601_133700-9x12ylug/files/code/train/rl_llava_next_video.py @@ -0,0 +1,358 @@ +import copy +import os +import sys +from typing import Iterable, List +import warnings +import random + +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import DataLoader, Dataset +import wandb +from accelerate import Accelerator +from omegaconf import OmegaConf + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LLAVA_NEXT_ROOT = os.path.join(REPO_ROOT, "LLaVA-NeXT") +sys.path.insert(0, LLAVA_NEXT_ROOT) +sys.path.insert(1, REPO_ROOT) + +from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX +from llava.conversation import conv_templates +from llava.mm_utils import tokenizer_image_token +from llava.model.builder import load_pretrained_model +from llava.utils import disable_torch_init, process_video_with_decord + +from train.llava_next_video.common import ( + build_video_mcq_prompt, + ensure_dir, + get_config, + load_json, + resolve_media_abs_path, +) +from train.utils import ensure_wandb_run_id, flatten_omega_conf + +warnings.filterwarnings("ignore", message=".*copying from a non-meta parameter.*") +warnings.filterwarnings("ignore", message=".*Flash Attention 2.0 with a model not initialized on GPU.*") + + +class RLDataset(Dataset): + def __init__(self, samples): + self.samples = samples + + def __len__(self): + return len(self.samples) + + def __getitem__(self, idx): + return self.samples[idx] + + +def build_prompt_text(sample: dict) -> str: + return build_video_mcq_prompt(sample) + + +def tokenize_example(tokenizer, conv_mode: str, prompt_text: str, response_text: str): + prompt_conv = copy.deepcopy(conv_templates[conv_mode]) + prompt_conv.append_message(prompt_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + prompt_conv.append_message(prompt_conv.roles[1], None) + prompt_ids = tokenizer_image_token(prompt_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + full_conv = copy.deepcopy(conv_templates[conv_mode]) + full_conv.append_message(full_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + full_conv.append_message(full_conv.roles[1], response_text) + full_ids = tokenizer_image_token(full_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + labels = full_ids.clone() + labels[: prompt_ids.numel()] = -100 + return full_ids, labels + + +def load_video_inputs(sample: dict, image_processor, model_cfg, device): + video_path = resolve_media_abs_path(sample, None, media_type="video") + video, _, _, _ = process_video_with_decord(video_path, model_cfg) + pixel_values = image_processor.preprocess(video, return_tensors="pt")["pixel_values"].to(device=device, dtype=torch.float16) + return [pixel_values], [video[0].size], ["video"] + + +def compute_token_logps(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: + shifted_logits = logits[:, :-1, :] + shifted_labels = labels[:, 1:] + valid_mask = shifted_labels.ne(-100) + gather_labels = shifted_labels.masked_fill(~valid_mask, 0) + token_logps = F.log_softmax(shifted_logits, dim=-1).gather(-1, gather_labels.unsqueeze(-1)).squeeze(-1) + token_logps = token_logps[valid_mask] + return token_logps[torch.isfinite(token_logps)] + + +def safe_mean(values: torch.Tensor, *, device: torch.device, dtype: torch.dtype) -> torch.Tensor: + if values.numel() == 0: + return torch.zeros((), device=device, dtype=dtype) + return values.mean() + + +def snapshot_named_parameters(model, names: Iterable[str]) -> dict[str, torch.Tensor]: + state = {} + target_names = set(names) + for name, param in model.named_parameters(): + if name in target_names: + state[name] = param.detach().cpu().clone() + return state + + +def restore_named_parameters(model, state: dict[str, torch.Tensor]) -> None: + for name, param in model.named_parameters(): + if name in state: + param.data.copy_(state[name].to(device=param.device, dtype=param.dtype)) + + +def gradients_are_finite(params: Iterable[torch.nn.Parameter]) -> bool: + for param in params: + if param.grad is not None and not torch.isfinite(param.grad).all(): + return False + return True + + +def parameters_are_finite(params: Iterable[torch.nn.Parameter]) -> bool: + return all(torch.isfinite(param).all() for param in params) + + +def _merge_overwrite_config(config) -> dict: + overwrite_config = OmegaConf.to_container(config.model.get("load_overwrite_config", {}), resolve=True) or {} + training_overrides = OmegaConf.to_container(config.training.get("load_overwrite_config", {}), resolve=True) or {} + overwrite_config.update(training_overrides) + return overwrite_config + + +def _set_trainable_parameters(model, trainable_parts: Iterable[str]) -> List[torch.nn.Parameter]: + part_to_patterns = { + "lm_head": ["lm_head"], + "embed_tokens": ["embed_tokens"], + "mm_projector": ["mm_projector"], + "vision_resampler": ["vision_resampler"], + "vision_tower": ["vision_tower"], + } + enabled_parts = list(trainable_parts) + enabled_patterns = [] + for part in enabled_parts: + if part == "all": + for param in model.parameters(): + param.requires_grad_(True) + return list(model.parameters()) + enabled_patterns.extend(part_to_patterns.get(part, [part])) + + trainable_params: List[torch.nn.Parameter] = [] + seen_param_ids = set() + for name, param in model.named_parameters(): + should_train = any(pattern in name for pattern in enabled_patterns) + param.requires_grad_(should_train) + if should_train and id(param) not in seen_param_ids: + trainable_params.append(param) + seen_param_ids.add(id(param)) + return trainable_params + + +def _summarize_trainable_parameters(model) -> tuple[int, int, List[str]]: + total_params = 0 + trainable_params = 0 + trainable_names = [] + for name, param in model.named_parameters(): + count = param.numel() + total_params += count + if param.requires_grad: + trainable_params += count + trainable_names.append(name) + return trainable_params, total_params, trainable_names + + +def set_seed(seed: int) -> None: + random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + + +if __name__ == "__main__": + config = get_config() + disable_torch_init() + set_seed(int(config.training.get("seed", 10086))) + if hasattr(torch.backends, "cuda") and torch.cuda.is_available(): + torch.backends.cuda.matmul.allow_tf32 = bool(config.training.get("enable_tf32", True)) + if hasattr(torch.backends, "cudnn"): + torch.backends.cudnn.allow_tf32 = bool(config.training.get("enable_tf32", True)) + + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + ) + + existing_run_id = ensure_wandb_run_id(config) + if accelerator.is_main_process: + wandb.init( + project=config.experiment.project, + id=existing_run_id, + resume=config.wandb.resume, + config={k: v for k, v in flatten_omega_conf(config, resolve=True)}, + name=config.experiment.project, + reinit=True, + ) + + current_epoch = int(config.experiment.current_epoch) + model_path = config.model.pretrained_model if current_epoch == 1 else os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ref_model_path = config.model.reference_model or config.model.pretrained_model + train_samples = load_json(os.path.join(config.experiment.project, "temp_data", config.dataset.optimization_data_file)) + overwrite_config = _merge_overwrite_config(config) + ref_device = torch.device("cuda:1") if torch.cuda.is_available() and torch.cuda.device_count() > 1 else accelerator.device + if accelerator.is_main_process: + print( + f"[train] epoch={current_epoch} samples={len(train_samples)} " + f"grad_accum={int(config.training.gradient_accumulation_steps)}", + flush=True, + ) + + tokenizer, model, image_processor, _ = load_pretrained_model( + model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + _, ref_model, _, _ = load_pretrained_model( + ref_model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + ref_model.to(device=ref_device, dtype=torch.float16) + ref_model.eval() + for p in ref_model.parameters(): + p.requires_grad_(False) + + trainable_parts = list(config.training.get("trainable_parts", ["lm_head"])) + trainable_params = _set_trainable_parameters(model, trainable_parts) + if not trainable_params: + raise ValueError(f"No trainable parameters matched training.trainable_parts={trainable_parts}") + + trainable_count, total_count, trainable_names = _summarize_trainable_parameters(model) + if accelerator.is_main_process: + print( + f"RL trainable parts: {trainable_parts}; " + f"trainable params={trainable_count:,}/{total_count:,} " + f"({100.0 * trainable_count / max(total_count, 1):.4f}%)" + ) + for name in trainable_names[:20]: + print(f" trainable: {name}") + if len(trainable_names) > 20: + print(f" ... and {len(trainable_names) - 20} more") + + optimizer = AdamW(trainable_params, lr=float(config.optimizer.learning_rate), weight_decay=float(config.optimizer.weight_decay)) + dataloader = DataLoader(RLDataset(train_samples), batch_size=1, shuffle=True) + + model, optimizer, dataloader = accelerator.prepare(model, optimizer, dataloader) + last_good_state = snapshot_named_parameters(accelerator.unwrap_model(model), trainable_names) + model.train() + + global_step = 0 + skipped_steps = 0 + for _ in range(int(config.training.num_train_epochs)): + for batch in dataloader: + sample = {k: v[0] if isinstance(v, list) else v for k, v in batch.items()} + prompt_text = build_prompt_text(sample) + input_ids, labels = tokenize_example(tokenizer, config.model.conv_mode, prompt_text, sample["response"]) + input_ids = input_ids.unsqueeze(0).to(accelerator.device) + labels = labels.unsqueeze(0).to(accelerator.device) + attention_mask = input_ids.ne(tokenizer.pad_token_id if tokenizer.pad_token_id is not None else 0).long() + images, image_sizes, modalities = load_video_inputs(sample, image_processor, model.config, accelerator.device) + ref_input_ids = input_ids.to(ref_device) + ref_attention_mask = attention_mask.to(ref_device) + ref_images = [img.to(ref_device) for img in images] + ref_labels = labels.to(ref_device) + + with accelerator.accumulate(model): + outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + images=images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + with torch.no_grad(): + ref_outputs = ref_model( + input_ids=ref_input_ids, + attention_mask=ref_attention_mask, + images=ref_images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + + policy_logps = compute_token_logps(outputs.logits, labels) + ref_logps = compute_token_logps(ref_outputs.logits, ref_labels).to(policy_logps.device) + shared_tokens = min(policy_logps.numel(), ref_logps.numel()) + if shared_tokens == 0: + skipped_steps += 1 + optimizer.zero_grad() + continue + policy_logps = policy_logps[:shared_tokens] + ref_logps = ref_logps[:shared_tokens] + advantage = torch.tensor(float(sample["reward"]), device=accelerator.device, dtype=policy_logps.dtype) + pg_loss = -(advantage * safe_mean(policy_logps, device=accelerator.device, dtype=policy_logps.dtype)) + kl_loss = torch.clamp( + safe_mean(policy_logps - ref_logps, device=accelerator.device, dtype=policy_logps.dtype), + min=0.0, + ) + loss = pg_loss + float(config.training.kl_beta) * kl_loss + if not torch.isfinite(loss): + skipped_steps += 1 + optimizer.zero_grad() + continue + + accelerator.backward(loss) + if not gradients_are_finite(trainable_params): + skipped_steps += 1 + optimizer.zero_grad() + restore_named_parameters(accelerator.unwrap_model(model), last_good_state) + continue + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, float(config.training.max_grad_norm)) + optimizer.step() + if not parameters_are_finite(trainable_params): + skipped_steps += 1 + restore_named_parameters(accelerator.unwrap_model(model), last_good_state) + optimizer.zero_grad() + continue + last_good_state = snapshot_named_parameters(accelerator.unwrap_model(model), trainable_names) + optimizer.zero_grad() + + global_step += 1 + if accelerator.is_main_process and global_step % int(config.training.logging_steps) == 0: + print( + f"[train] epoch={current_epoch} step={global_step} " + f"loss={float(loss.detach().cpu()):.6f} " + f"pg={float(pg_loss.detach().cpu()):.6f} " + f"kl={float(kl_loss.detach().cpu()):.6f} " + f"reward={float(advantage.detach().cpu()):.6f} " + f"skipped={skipped_steps}", + flush=True, + ) + wandb.log( + { + "train/loss": float(loss.detach().cpu()), + "train/pg_loss": float(pg_loss.detach().cpu()), + "train/kl_loss": float(kl_loss.detach().cpu()), + "train/advantage": float(advantage.detach().cpu()), + "train/skipped_steps": skipped_steps, + "train/step": global_step, + } + ) + + accelerator.wait_for_everyone() + if accelerator.is_main_process: + out_dir = os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ensure_dir(out_dir) + accelerator.unwrap_model(model).save_pretrained(out_dir, safe_serialization=True) + tokenizer.save_pretrained(out_dir) + print(f"[train] epoch={current_epoch} checkpoint_saved={os.path.abspath(out_dir)}", flush=True) + wandb.finish() diff --git a/wandb/run-20260601_133700-9x12ylug/files/config.yaml b/wandb/run-20260601_133700-9x12ylug/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a8d3e897be20dc0e03a79a40f8ae8aada1097c9 --- /dev/null +++ b/wandb/run-20260601_133700-9x12ylug/files/config.yaml @@ -0,0 +1,223 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_llava_next_video.py + e: + f2it9tpacxjug20nb037ehmm1thy422p: + args: + - config=configs/rl_llava_next_video_reversal.yaml + - wandb.run_id=9x12ylug + - experiment.current_epoch=8 + codePath: train/rl_llava_next_video.py + codePathLocal: train/rl_llava_next_video.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "626052399104" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/llava-next-video/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_llava_next_video.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-06-01T05:37:00.841096Z" + writerId: f2it9tpacxjug20nb037ehmm1thy422p + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + "2": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.40.0.dev0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_llava_next_video_reversal.yaml +dataset.optimization_data_file: + value: rl_data.json +dataset.train_json: + value: /data4/runqing/dLLM-RL/data/video_reversal_rl_train.json +evaluation.eval_json: + value: /data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json +evaluation.max_new_tokens: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.num_task_per_step: + value: 64 +evaluation.temperature: + value: 0.1 +experiment.current_epoch: + value: 8 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.project: + value: rl_llava_next_video_reversal +experiment.save_every: + value: 10 +experiment.total_step: + value: 30 +model.conv_mode: + value: vicuna_v1 +model.load_overwrite_config.add_time_instruction: + value: true +model.load_overwrite_config.force_sample: + value: true +model.load_overwrite_config.frames_upbound: + value: 32 +model.load_overwrite_config.max_sequence_length: + value: 8192 +model.load_overwrite_config.mm_newline_position: + value: no_token +model.load_overwrite_config.mm_spatial_pool_mode: + value: average +model.load_overwrite_config.mm_spatial_pool_stride: + value: 2 +model.load_overwrite_config.mm_vision_tower: + value: /data4/runqing/hf/clip-vit-large-patch14-336 +model.load_overwrite_config.rope_scaling.factor: + value: 2 +model.load_overwrite_config.rope_scaling.type: + value: linear +model.load_overwrite_config.tokenizer_model_max_length: + value: 8192 +model.load_overwrite_config.video_fps: + value: 1 +model.model_base: + value: null +model.model_name: + value: llava_llama +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +model.reference_model: + value: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +optimizer.learning_rate: + value: 1e-06 +optimizer.weight_decay: + value: 0 +reward.direction_consistency_bonus: + value: 0.3 +reward.max_direction_gap: + value: 0.2 +rollout.max_new_tokens: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.temperature: + value: 0.8 +rollout.top_p: + value: 0.95 +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/llava-next-video +training.enable_tf32: + value: true +training.gradient_accumulation_steps: + value: 2 +training.kl_beta: + value: 0.01 +training.load_overwrite_config.frames_upbound: + value: 8 +training.load_overwrite_config.max_sequence_length: + value: 2048 +training.load_overwrite_config.tokenizer_model_max_length: + value: 2048 +training.logging_steps: + value: 1 +training.max_grad_norm: + value: 1 +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.seed: + value: 10086 +training.trainable_parts.0: + value: mm_projector +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: 9x12ylug diff --git a/wandb/run-20260601_133700-9x12ylug/files/diff.patch b/wandb/run-20260601_133700-9x12ylug/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260601_133700-9x12ylug/files/diff.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260601_133700-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260601_133700-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260601_133700-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260601_133700-9x12ylug/files/output.log b/wandb/run-20260601_133700-9x12ylug/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..116828d4e2df4eb8f1e2ce9c605f28e4f2b0ade0 --- /dev/null +++ b/wandb/run-20260601_133700-9x12ylug/files/output.log @@ -0,0 +1,529 @@ +[train] epoch=8 samples=1024 grad_accum=2 +Loaded LLaVA model: rl_llava_next_video_reversal/ckpt/optimized +Overwriting config with {'mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'mm_spatial_pool_stride': 2, 'mm_spatial_pool_mode': 'average', 'mm_newline_position': 'no_token', 'max_sequence_length': 2048, 'tokenizer_model_max_length': 2048, 'rope_scaling': {'factor': 2.0, 'type': 'linear'}, 'video_fps': 1, 'frames_upbound': 8, 'force_sample': True, 'add_time_instruction': True} +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +The checkpoint seems to contain `vision_tower` weights: `mm_tunable_parts` contains `mm_vision_tower`. +Model Class: LlavaLlamaForCausalLM +Loaded LLaVA model: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +You are using a model of type llava to instantiate a model of type llava_llama. This is not supported for all configurations of models and can yield errors. +Overwriting config with {'mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'mm_spatial_pool_stride': 2, 'mm_spatial_pool_mode': 'average', 'mm_newline_position': 'no_token', 'max_sequence_length': 2048, 'tokenizer_model_max_length': 2048, 'rope_scaling': {'factor': 2.0, 'type': 'linear'}, 'video_fps': 1, 'frames_upbound': 8, 'force_sample': True, 'add_time_instruction': True} +The checkpoint seems to contain `vision_tower` weights: `mm_tunable_parts` contains `mm_vision_tower`. +Model Class: LlavaLlamaForCausalLM +RL trainable parts: ['mm_projector']; trainable params=20,979,712/7,062,915,072 (0.2970%) + trainable: model.mm_projector.0.weight + trainable: model.mm_projector.0.bias + trainable: model.mm_projector.2.weight + trainable: model.mm_projector.2.bias +[train] epoch=8 step=1 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=0 +[train] epoch=8 step=2 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=1 +[train] epoch=8 step=3 loss=6.282383 pg=6.282336 kl=0.004788 reward=1.000000 skipped=2 +[train] epoch=8 step=4 loss=6.468365 pg=6.468350 kl=0.001409 reward=1.000000 skipped=3 +[train] epoch=8 step=5 loss=7.083083 pg=7.083083 kl=0.000000 reward=1.000000 skipped=4 +[train] epoch=8 step=6 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=5 +[train] epoch=8 step=7 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=6 +[train] epoch=8 step=8 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=7 +[train] epoch=8 step=9 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=8 +[train] epoch=8 step=10 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=9 +[train] epoch=8 step=11 loss=-8.891484 pg=-8.891523 kl=0.003878 reward=-1.732051 skipped=10 +[train] epoch=8 step=12 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=11 +[train] epoch=8 step=13 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=12 +[train] epoch=8 step=14 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=13 +[train] epoch=8 step=15 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=14 +[train] epoch=8 step=16 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=15 +[train] epoch=8 step=17 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=16 +[train] epoch=8 step=18 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=17 +[train] epoch=8 step=19 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=18 +[train] epoch=8 step=20 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=19 +[train] epoch=8 step=21 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=20 +[train] epoch=8 step=22 loss=7.845365 pg=7.845365 kl=0.000000 reward=1.000000 skipped=21 +[train] epoch=8 step=23 loss=7.083083 pg=7.083083 kl=0.000000 reward=1.000000 skipped=22 +[train] epoch=8 step=24 loss=5.951652 pg=5.951652 kl=0.000000 reward=1.000000 skipped=23 +[train] epoch=8 step=25 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=24 +[train] epoch=8 step=26 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=25 +[train] epoch=8 step=27 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=26 +[train] epoch=8 step=28 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=27 +[train] epoch=8 step=29 loss=4.836777 pg=4.836777 kl=0.000000 reward=1.000000 skipped=28 +[train] epoch=8 step=30 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=29 +[train] epoch=8 step=31 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=30 +[train] epoch=8 step=32 loss=5.878745 pg=5.878745 kl=0.000000 reward=1.000000 skipped=31 +[train] epoch=8 step=33 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=32 +[train] epoch=8 step=34 loss=7.771910 pg=7.771876 kl=0.003357 reward=1.000000 skipped=33 +[train] epoch=8 step=35 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=34 +[train] epoch=8 step=36 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=35 +[train] epoch=8 step=37 loss=8.125317 pg=8.125317 kl=0.000000 reward=1.000000 skipped=36 +[train] epoch=8 step=38 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=37 +[train] epoch=8 step=39 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=38 +[train] epoch=8 step=40 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=39 +[train] epoch=8 step=41 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=40 +[train] epoch=8 step=42 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=41 +[train] epoch=8 step=43 loss=-8.891484 pg=-8.891523 kl=0.003878 reward=-1.732051 skipped=42 +[train] epoch=8 step=44 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=43 +[train] epoch=8 step=45 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=44 +[train] epoch=8 step=46 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=45 +[train] epoch=8 step=47 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=46 +[train] epoch=8 step=48 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=47 +[train] epoch=8 step=49 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=48 +[train] epoch=8 step=50 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=49 +[train] epoch=8 step=51 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=50 +[train] epoch=8 step=52 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=51 +[train] epoch=8 step=53 loss=4.328114 pg=4.328114 kl=0.000000 reward=0.774597 skipped=52 +[train] epoch=8 step=54 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=53 +[train] epoch=8 step=55 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=54 +[train] epoch=8 step=56 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=55 +[train] epoch=8 step=57 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=56 +[train] epoch=8 step=58 loss=-6.788513 pg=-6.788513 kl=0.000000 reward=-1.000000 skipped=57 +[train] epoch=8 step=59 loss=5.369410 pg=5.369405 kl=0.000436 reward=1.000000 skipped=58 +[train] epoch=8 step=60 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=59 +[train] epoch=8 step=61 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=60 +[train] epoch=8 step=62 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=61 +[train] epoch=8 step=63 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=62 +[train] epoch=8 step=64 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=63 +[train] epoch=8 step=65 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=64 +[train] epoch=8 step=66 loss=5.436614 pg=5.436614 kl=0.000000 reward=1.000000 skipped=65 +[train] epoch=8 step=67 loss=5.568174 pg=5.568095 kl=0.007986 reward=1.000000 skipped=66 +[train] epoch=8 step=68 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=67 +[train] epoch=8 step=69 loss=6.862856 pg=6.862832 kl=0.002382 reward=1.000000 skipped=68 +[train] epoch=8 step=70 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=69 +[train] epoch=8 step=71 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=70 +[train] epoch=8 step=72 loss=6.468365 pg=6.468350 kl=0.001409 reward=1.000000 skipped=71 +[train] epoch=8 step=73 loss=8.117537 pg=8.117537 kl=0.000057 reward=1.000000 skipped=72 +[train] epoch=8 step=74 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=73 +[train] epoch=8 step=75 loss=8.125317 pg=8.125317 kl=0.000000 reward=1.000000 skipped=74 +[train] epoch=8 step=76 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=75 +[train] epoch=8 step=77 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=76 +[train] epoch=8 step=78 loss=5.282966 pg=5.282931 kl=0.003500 reward=1.000000 skipped=77 +[train] epoch=8 step=79 loss=5.875602 pg=5.875589 kl=0.001239 reward=1.000000 skipped=78 +[train] epoch=8 step=80 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=79 +[train] epoch=8 step=81 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=80 +[train] epoch=8 step=82 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=81 +[train] epoch=8 step=83 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=82 +[train] epoch=8 step=84 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=83 +[train] epoch=8 step=85 loss=7.771910 pg=7.771876 kl=0.003357 reward=1.000000 skipped=84 +[train] epoch=8 step=86 loss=6.194232 pg=6.194232 kl=0.000000 reward=1.000000 skipped=85 +[train] epoch=8 step=87 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=86 +[train] epoch=8 step=88 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=87 +[train] epoch=8 step=89 loss=4.836777 pg=4.836777 kl=0.000000 reward=1.000000 skipped=88 +[train] epoch=8 step=90 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=89 +[train] epoch=8 step=91 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=90 +[train] epoch=8 step=92 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=91 +[train] epoch=8 step=93 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=92 +[train] epoch=8 step=94 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=93 +[train] epoch=8 step=95 loss=7.021185 pg=7.021152 kl=0.003361 reward=1.000000 skipped=94 +[train] epoch=8 step=96 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=95 +[train] epoch=8 step=97 loss=7.256117 pg=7.256107 kl=0.000982 reward=1.000000 skipped=96 +[train] epoch=8 step=98 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=97 +[train] epoch=8 step=99 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=98 +[train] epoch=8 step=100 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=99 +[train] epoch=8 step=101 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=100 +[train] epoch=8 step=102 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=101 +[train] epoch=8 step=103 loss=6.126596 pg=6.126578 kl=0.001824 reward=1.000000 skipped=102 +[train] epoch=8 step=104 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=103 +[train] epoch=8 step=105 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=104 +[train] epoch=8 step=106 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=105 +[train] epoch=8 step=107 loss=6.295836 pg=6.295836 kl=0.000000 reward=1.000000 skipped=106 +[train] epoch=8 step=108 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=107 +[train] epoch=8 step=109 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=108 +[train] epoch=8 step=110 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=109 +[train] epoch=8 step=111 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=110 +[train] epoch=8 step=112 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=111 +[train] epoch=8 step=113 loss=6.295836 pg=6.295836 kl=0.000000 reward=1.000000 skipped=112 +[train] epoch=8 step=114 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=113 +[train] epoch=8 step=115 loss=6.295836 pg=6.295836 kl=0.000000 reward=1.000000 skipped=114 +[train] epoch=8 step=116 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=115 +[train] epoch=8 step=117 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=116 +[train] epoch=8 step=118 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=117 +[train] epoch=8 step=119 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=118 +[train] epoch=8 step=120 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=119 +[train] epoch=8 step=121 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=120 +[train] epoch=8 step=122 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=121 +[train] epoch=8 step=123 loss=7.845365 pg=7.845365 kl=0.000000 reward=1.000000 skipped=122 +[train] epoch=8 step=124 loss=6.414127 pg=6.414127 kl=0.000000 reward=1.000000 skipped=123 +[train] epoch=8 step=125 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=124 +[train] epoch=8 step=126 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=125 +[train] epoch=8 step=127 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=126 +[train] epoch=8 step=128 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=127 +[train] epoch=8 step=129 loss=6.862856 pg=6.862832 kl=0.002382 reward=1.000000 skipped=128 +[train] epoch=8 step=130 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=129 +[train] epoch=8 step=131 loss=7.021185 pg=7.021152 kl=0.003361 reward=1.000000 skipped=130 +[train] epoch=8 step=132 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=131 +[train] epoch=8 step=133 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=132 +[train] epoch=8 step=134 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=133 +[train] epoch=8 step=135 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=134 +[train] epoch=8 step=136 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=135 +[train] epoch=8 step=137 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=136 +[train] epoch=8 step=138 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=137 +[train] epoch=8 step=139 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=138 +[train] epoch=8 step=140 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=139 +[train] epoch=8 step=141 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=140 +[train] epoch=8 step=142 loss=5.734613 pg=5.734613 kl=0.000000 reward=1.000000 skipped=141 +[train] epoch=8 step=143 loss=5.878745 pg=5.878745 kl=0.000000 reward=1.000000 skipped=142 +[train] epoch=8 step=144 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=143 +[train] epoch=8 step=145 loss=-7.987633 pg=-7.987633 kl=0.000000 reward=-1.290994 skipped=144 +[train] epoch=8 step=146 loss=6.068417 pg=6.068375 kl=0.004213 reward=1.000000 skipped=145 +[train] epoch=8 step=147 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=146 +[train] epoch=8 step=148 loss=8.117537 pg=8.117537 kl=0.000057 reward=1.000000 skipped=147 +[train] epoch=8 step=149 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=148 +[train] epoch=8 step=150 loss=5.436614 pg=5.436614 kl=0.000000 reward=1.000000 skipped=149 +[train] epoch=8 step=151 loss=2.614115 pg=2.614115 kl=0.000000 reward=0.377964 skipped=150 +[train] epoch=8 step=152 loss=5.446908 pg=5.446908 kl=0.000000 reward=1.000000 skipped=151 +[train] epoch=8 step=153 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=152 +[train] epoch=8 step=154 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=153 +[train] epoch=8 step=155 loss=2.389125 pg=2.389019 kl=0.010641 reward=0.377964 skipped=154 +[train] epoch=8 step=156 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=155 +[train] epoch=8 step=157 loss=5.592641 pg=5.592641 kl=0.000000 reward=1.000000 skipped=156 +[train] epoch=8 step=158 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=157 +[train] epoch=8 step=159 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=158 +[train] epoch=8 step=160 loss=0.000010 pg=0.000000 kl=0.000967 reward=0.000000 skipped=159 +[train] epoch=8 step=161 loss=6.414127 pg=6.414127 kl=0.000000 reward=1.000000 skipped=160 +[train] epoch=8 step=162 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=161 +[train] epoch=8 step=163 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=162 +[train] epoch=8 step=164 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=163 +[train] epoch=8 step=165 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=164 +[train] epoch=8 step=166 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=165 +[train] epoch=8 step=167 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=166 +[train] epoch=8 step=168 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=167 +[train] epoch=8 step=169 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=168 +[train] epoch=8 step=170 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=169 +[train] epoch=8 step=171 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=170 +[train] epoch=8 step=172 loss=5.438523 pg=5.438523 kl=0.000000 reward=1.000000 skipped=171 +[train] epoch=8 step=173 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=172 +[train] epoch=8 step=174 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=173 +[train] epoch=8 step=175 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=174 +[train] epoch=8 step=176 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=175 +[train] epoch=8 step=177 loss=2.614115 pg=2.614115 kl=0.000000 reward=0.377964 skipped=176 +[train] epoch=8 step=178 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=177 +[train] epoch=8 step=179 loss=5.282966 pg=5.282931 kl=0.003500 reward=1.000000 skipped=178 +[train] epoch=8 step=180 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=179 +[train] epoch=8 step=181 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=180 +[train] epoch=8 step=182 loss=2.614115 pg=2.614115 kl=0.000000 reward=0.377964 skipped=181 +[train] epoch=8 step=183 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=182 +[train] epoch=8 step=184 loss=6.414127 pg=6.414127 kl=0.000000 reward=1.000000 skipped=183 +[train] epoch=8 step=185 loss=4.706718 pg=4.706718 kl=0.000000 reward=1.000000 skipped=184 +[train] epoch=8 step=186 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=185 +[train] epoch=8 step=187 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=186 +[train] epoch=8 step=188 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=187 +[train] epoch=8 step=189 loss=5.438523 pg=5.438523 kl=0.000000 reward=1.000000 skipped=188 +[train] epoch=8 step=190 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=189 +[train] epoch=8 step=191 loss=7.161344 pg=7.161344 kl=0.000000 reward=1.000000 skipped=190 +[train] epoch=8 step=192 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=191 +[train] epoch=8 step=193 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=192 +[train] epoch=8 step=194 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=193 +[train] epoch=8 step=195 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=194 +[train] epoch=8 step=196 loss=2.496069 pg=2.496067 kl=0.000205 reward=0.377964 skipped=195 +[train] epoch=8 step=197 loss=-5.594396 pg=-5.594400 kl=0.000427 reward=-1.000000 skipped=196 +[train] epoch=8 step=198 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=197 +[train] epoch=8 step=199 loss=-5.402396 pg=-5.402436 kl=0.004019 reward=-1.000000 skipped=198 +[train] epoch=8 step=200 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=199 +[train] epoch=8 step=201 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=200 +[train] epoch=8 step=202 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=201 +[train] epoch=8 step=203 loss=2.389125 pg=2.389019 kl=0.010641 reward=0.377964 skipped=202 +[train] epoch=8 step=204 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=203 +[train] epoch=8 step=205 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=204 +[train] epoch=8 step=206 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=205 +[train] epoch=8 step=207 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=206 +[train] epoch=8 step=208 loss=6.468365 pg=6.468350 kl=0.001409 reward=1.000000 skipped=207 +[train] epoch=8 step=209 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=208 +[train] epoch=8 step=210 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=209 +[train] epoch=8 step=211 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=210 +[train] epoch=8 step=212 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=211 +[train] epoch=8 step=213 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=212 +[train] epoch=8 step=214 loss=2.496069 pg=2.496067 kl=0.000205 reward=0.377964 skipped=213 +[train] epoch=8 step=215 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=214 +[train] epoch=8 step=216 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=215 +[train] epoch=8 step=217 loss=6.068417 pg=6.068375 kl=0.004213 reward=1.000000 skipped=216 +[train] epoch=8 step=218 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=217 +[train] epoch=8 step=219 loss=5.581098 pg=5.581098 kl=0.000000 reward=1.000000 skipped=218 +[train] epoch=8 step=220 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=219 +[train] epoch=8 step=221 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=220 +[train] epoch=8 step=222 loss=3.049087 pg=3.049040 kl=0.004637 reward=0.577350 skipped=221 +[train] epoch=8 step=223 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=222 +[train] epoch=8 step=224 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=223 +[train] epoch=8 step=225 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=224 +[train] epoch=8 step=226 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=225 +[train] epoch=8 step=227 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=226 +[train] epoch=8 step=228 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=227 +[train] epoch=8 step=229 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=228 +[train] epoch=8 step=230 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=229 +[train] epoch=8 step=231 loss=6.120806 pg=6.120743 kl=0.006336 reward=1.000000 skipped=230 +[train] epoch=8 step=232 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=231 +[train] epoch=8 step=233 loss=2.496069 pg=2.496067 kl=0.000205 reward=0.377964 skipped=232 +[train] epoch=8 step=234 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=233 +[train] epoch=8 step=235 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=234 +[train] epoch=8 step=236 loss=6.746428 pg=6.746428 kl=0.000000 reward=1.000000 skipped=235 +[train] epoch=8 step=237 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=236 +[train] epoch=8 step=238 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=237 +[train] epoch=8 step=239 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=238 +[train] epoch=8 step=240 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=239 +[train] epoch=8 step=241 loss=5.446908 pg=5.446908 kl=0.000000 reward=1.000000 skipped=240 +[train] epoch=8 step=242 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=241 +[train] epoch=8 step=243 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=242 +[train] epoch=8 step=244 loss=8.125317 pg=8.125317 kl=0.000000 reward=1.000000 skipped=243 +[train] epoch=8 step=245 loss=7.926152 pg=7.926152 kl=0.000000 reward=1.000000 skipped=244 +[train] epoch=8 step=246 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=245 +[train] epoch=8 step=247 loss=7.161344 pg=7.161344 kl=0.000000 reward=1.000000 skipped=246 +[train] epoch=8 step=248 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=247 +[train] epoch=8 step=249 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=248 +[train] epoch=8 step=250 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=249 +[train] epoch=8 step=251 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=250 +[train] epoch=8 step=252 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=251 +[train] epoch=8 step=253 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=252 +[train] epoch=8 step=254 loss=5.592641 pg=5.592641 kl=0.000000 reward=1.000000 skipped=253 +[train] epoch=8 step=255 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=254 +[train] epoch=8 step=256 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=255 +[train] epoch=8 step=257 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=256 +[train] epoch=8 step=258 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=257 +[train] epoch=8 step=259 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=258 +[train] epoch=8 step=260 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=259 +[train] epoch=8 step=261 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=260 +[train] epoch=8 step=262 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=261 +[train] epoch=8 step=263 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=262 +[train] epoch=8 step=264 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=263 +[train] epoch=8 step=265 loss=2.389125 pg=2.389019 kl=0.010641 reward=0.377964 skipped=264 +[train] epoch=8 step=266 loss=-17.953377 pg=-17.953447 kl=0.007016 reward=-2.645751 skipped=265 +[train] epoch=8 step=267 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=266 +[train] epoch=8 step=268 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=267 +[train] epoch=8 step=269 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=268 +[train] epoch=8 step=270 loss=4.706718 pg=4.706718 kl=0.000000 reward=1.000000 skipped=269 +[train] epoch=8 step=271 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=270 +[train] epoch=8 step=272 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=271 +[train] epoch=8 step=273 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=272 +[train] epoch=8 step=274 loss=2.496069 pg=2.496067 kl=0.000205 reward=0.377964 skipped=273 +[train] epoch=8 step=275 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=274 +[train] epoch=8 step=276 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=275 +[train] epoch=8 step=277 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=276 +[train] epoch=8 step=278 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=277 +[train] epoch=8 step=279 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=278 +[train] epoch=8 step=280 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=279 +[train] epoch=8 step=281 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=280 +[train] epoch=8 step=282 loss=5.875602 pg=5.875589 kl=0.001239 reward=1.000000 skipped=281 +[train] epoch=8 step=283 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=282 +[train] epoch=8 step=284 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=283 +[train] epoch=8 step=285 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=284 +[train] epoch=8 step=286 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=285 +[train] epoch=8 step=287 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=286 +[train] epoch=8 step=288 loss=6.282383 pg=6.282336 kl=0.004788 reward=1.000000 skipped=287 +[train] epoch=8 step=289 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=288 +[train] epoch=8 step=290 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=289 +[train] epoch=8 step=291 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=290 +[train] epoch=8 step=292 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=291 +[train] epoch=8 step=293 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=292 +[train] epoch=8 step=294 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=293 +[train] epoch=8 step=295 loss=7.314037 pg=7.314037 kl=0.000000 reward=1.000000 skipped=294 +[train] epoch=8 step=296 loss=6.120806 pg=6.120743 kl=0.006336 reward=1.000000 skipped=295 +[train] epoch=8 step=297 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=296 +[train] epoch=8 step=298 loss=0.000013 pg=0.000000 kl=0.001251 reward=0.000000 skipped=297 +[train] epoch=8 step=299 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=298 +[train] epoch=8 step=300 loss=0.000010 pg=0.000000 kl=0.000967 reward=0.000000 skipped=299 +[train] epoch=8 step=301 loss=6.862856 pg=6.862832 kl=0.002382 reward=1.000000 skipped=300 +[train] epoch=8 step=302 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=301 +[train] epoch=8 step=303 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=302 +[train] epoch=8 step=304 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=303 +[train] epoch=8 step=305 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=304 +[train] epoch=8 step=306 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=305 +[train] epoch=8 step=307 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=306 +[train] epoch=8 step=308 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=307 +[train] epoch=8 step=309 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=308 +[train] epoch=8 step=310 loss=4.706718 pg=4.706718 kl=0.000000 reward=1.000000 skipped=309 +[train] epoch=8 step=311 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=310 +[train] epoch=8 step=312 loss=-5.594396 pg=-5.594400 kl=0.000427 reward=-1.000000 skipped=311 +[train] epoch=8 step=313 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=312 +[train] epoch=8 step=314 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=313 +[train] epoch=8 step=315 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=314 +[train] epoch=8 step=316 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=315 +[train] epoch=8 step=317 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=316 +[train] epoch=8 step=318 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=317 +[train] epoch=8 step=319 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=318 +[train] epoch=8 step=320 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=319 +[train] epoch=8 step=321 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=320 +[train] epoch=8 step=322 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=321 +[train] epoch=8 step=323 loss=7.256117 pg=7.256107 kl=0.000982 reward=1.000000 skipped=322 +[train] epoch=8 step=324 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=323 +[train] epoch=8 step=325 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=324 +[train] epoch=8 step=326 loss=5.592641 pg=5.592641 kl=0.000000 reward=1.000000 skipped=325 +[train] epoch=8 step=327 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=326 +[train] epoch=8 step=328 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=327 +[train] epoch=8 step=329 loss=5.369410 pg=5.369405 kl=0.000436 reward=1.000000 skipped=328 +[train] epoch=8 step=330 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=329 +[train] epoch=8 step=331 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=330 +[train] epoch=8 step=332 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=331 +[train] epoch=8 step=333 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=332 +[train] epoch=8 step=334 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=333 +[train] epoch=8 step=335 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=334 +[train] epoch=8 step=336 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=335 +[train] epoch=8 step=337 loss=5.369410 pg=5.369405 kl=0.000436 reward=1.000000 skipped=336 +[train] epoch=8 step=338 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=337 +[train] epoch=8 step=339 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=338 +[train] epoch=8 step=340 loss=4.328114 pg=4.328114 kl=0.000000 reward=0.774597 skipped=339 +[train] epoch=8 step=341 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=340 +[train] epoch=8 step=342 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=341 +[train] epoch=8 step=343 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=342 +[train] epoch=8 step=344 loss=2.389125 pg=2.389019 kl=0.010641 reward=0.377964 skipped=343 +[train] epoch=8 step=345 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=344 +[train] epoch=8 step=346 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=345 +[train] epoch=8 step=347 loss=6.593108 pg=6.593024 kl=0.008324 reward=1.000000 skipped=346 +[train] epoch=8 step=348 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=347 +[train] epoch=8 step=349 loss=5.696458 pg=5.696399 kl=0.005970 reward=1.000000 skipped=348 +[train] epoch=8 step=350 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=349 +[train] epoch=8 step=351 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=350 +[train] epoch=8 step=352 loss=7.021185 pg=7.021152 kl=0.003361 reward=1.000000 skipped=351 +[train] epoch=8 step=353 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=352 +[train] epoch=8 step=354 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=353 +[train] epoch=8 step=355 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=354 +[train] epoch=8 step=356 loss=3.435869 pg=3.435869 kl=0.000000 reward=0.577350 skipped=355 +[train] epoch=8 step=357 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=356 +[train] epoch=8 step=358 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=357 +[train] epoch=8 step=359 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=358 +[train] epoch=8 step=360 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=359 +[train] epoch=8 step=361 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=360 +[train] epoch=8 step=362 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=361 +[train] epoch=8 step=363 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=362 +[train] epoch=8 step=364 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=363 +[train] epoch=8 step=365 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=364 +[train] epoch=8 step=366 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=365 +[train] epoch=8 step=367 loss=5.436614 pg=5.436614 kl=0.000000 reward=1.000000 skipped=366 +[train] epoch=8 step=368 loss=5.696458 pg=5.696399 kl=0.005970 reward=1.000000 skipped=367 +[train] epoch=8 step=369 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=368 +[train] epoch=8 step=370 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=369 +[train] epoch=8 step=371 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=370 +[train] epoch=8 step=372 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=371 +[train] epoch=8 step=373 loss=7.845365 pg=7.845365 kl=0.000000 reward=1.000000 skipped=372 +[train] epoch=8 step=374 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=373 +[train] epoch=8 step=375 loss=8.245316 pg=8.245316 kl=0.000000 reward=1.000000 skipped=374 +[train] epoch=8 step=376 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=375 +[train] epoch=8 step=377 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=376 +[train] epoch=8 step=378 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=377 +[train] epoch=8 step=379 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=378 +[train] epoch=8 step=380 loss=-11.342767 pg=-11.342767 kl=0.000000 reward=-1.732051 skipped=379 +[train] epoch=8 step=381 loss=6.194232 pg=6.194232 kl=0.000000 reward=1.000000 skipped=380 +[train] epoch=8 step=382 loss=5.951652 pg=5.951652 kl=0.000000 reward=1.000000 skipped=381 +[train] epoch=8 step=383 loss=6.194232 pg=6.194232 kl=0.000000 reward=1.000000 skipped=382 +[train] epoch=8 step=384 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=383 +[train] epoch=8 step=385 loss=4.836777 pg=4.836777 kl=0.000000 reward=1.000000 skipped=384 +[train] epoch=8 step=386 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=385 +[train] epoch=8 step=387 loss=7.204127 pg=7.204088 kl=0.003951 reward=1.000000 skipped=386 +[train] epoch=8 step=388 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=387 +[train] epoch=8 step=389 loss=5.734613 pg=5.734613 kl=0.000000 reward=1.000000 skipped=388 +[train] epoch=8 step=390 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=389 +[train] epoch=8 step=391 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=390 +[train] epoch=8 step=392 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=391 +[train] epoch=8 step=393 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=392 +[train] epoch=8 step=394 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=393 +[train] epoch=8 step=395 loss=5.446908 pg=5.446908 kl=0.000000 reward=1.000000 skipped=394 +[train] epoch=8 step=396 loss=6.173695 pg=6.173695 kl=0.000000 reward=1.000000 skipped=395 +[train] epoch=8 step=397 loss=3.049087 pg=3.049040 kl=0.004637 reward=0.577350 skipped=396 +[train] epoch=8 step=398 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=397 +[train] epoch=8 step=399 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=398 +[train] epoch=8 step=400 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=399 +[train] epoch=8 step=401 loss=5.734613 pg=5.734613 kl=0.000000 reward=1.000000 skipped=400 +[train] epoch=8 step=402 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=401 +[train] epoch=8 step=403 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=402 +[train] epoch=8 step=404 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=403 +[train] epoch=8 step=405 loss=6.120806 pg=6.120743 kl=0.006336 reward=1.000000 skipped=404 +[train] epoch=8 step=406 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=405 +[train] epoch=8 step=407 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=406 +[train] epoch=8 step=408 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=407 +[train] epoch=8 step=409 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=408 +[train] epoch=8 step=410 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=409 +[train] epoch=8 step=411 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=410 +[train] epoch=8 step=412 loss=6.578031 pg=6.578004 kl=0.002699 reward=1.000000 skipped=411 +[train] epoch=8 step=413 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=412 +[train] epoch=8 step=414 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=413 +[train] epoch=8 step=415 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=414 +[train] epoch=8 step=416 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=415 +[train] epoch=8 step=417 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=416 +[train] epoch=8 step=418 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=417 +[train] epoch=8 step=419 loss=2.614115 pg=2.614115 kl=0.000000 reward=0.377964 skipped=418 +[train] epoch=8 step=420 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=419 +[train] epoch=8 step=421 loss=0.000010 pg=0.000000 kl=0.000967 reward=0.000000 skipped=420 +[train] epoch=8 step=422 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=421 +[train] epoch=8 step=423 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=422 +[train] epoch=8 step=424 loss=6.746428 pg=6.746428 kl=0.000000 reward=1.000000 skipped=423 +[train] epoch=8 step=425 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=424 +[train] epoch=8 step=426 loss=2.614115 pg=2.614115 kl=0.000000 reward=0.377964 skipped=425 +[train] epoch=8 step=427 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=426 +[train] epoch=8 step=428 loss=5.438523 pg=5.438523 kl=0.000000 reward=1.000000 skipped=427 +[train] epoch=8 step=429 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=428 +[train] epoch=8 step=430 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=429 +[train] epoch=8 step=431 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=430 +[train] epoch=8 step=432 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=431 +[train] epoch=8 step=433 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=432 +[train] epoch=8 step=434 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=433 +[train] epoch=8 step=435 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=434 +[train] epoch=8 step=436 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=435 +[train] epoch=8 step=437 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=436 +[train] epoch=8 step=438 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=437 +[train] epoch=8 step=439 loss=7.161344 pg=7.161344 kl=0.000000 reward=1.000000 skipped=438 +[train] epoch=8 step=440 loss=7.312311 pg=7.312311 kl=0.000000 reward=1.000000 skipped=439 +[train] epoch=8 step=441 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=440 +[train] epoch=8 step=442 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=441 +[train] epoch=8 step=443 loss=7.204127 pg=7.204088 kl=0.003951 reward=1.000000 skipped=442 +[train] epoch=8 step=444 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=443 +[train] epoch=8 step=445 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=444 +[train] epoch=8 step=446 loss=5.696458 pg=5.696399 kl=0.005970 reward=1.000000 skipped=445 +[train] epoch=8 step=447 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=446 +[train] epoch=8 step=448 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=447 +[train] epoch=8 step=449 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=448 +[train] epoch=8 step=450 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=449 +[train] epoch=8 step=451 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=450 +[train] epoch=8 step=452 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=451 +[train] epoch=8 step=453 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=452 +[train] epoch=8 step=454 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=453 +[train] epoch=8 step=455 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=454 +[train] epoch=8 step=456 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=455 +[train] epoch=8 step=457 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=456 +[train] epoch=8 step=458 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=457 +[train] epoch=8 step=459 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=458 +[train] epoch=8 step=460 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=459 +[train] epoch=8 step=461 loss=-18.112778 pg=-18.112778 kl=0.000000 reward=-2.645751 skipped=460 +[train] epoch=8 step=462 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=461 +[train] epoch=8 step=463 loss=7.312311 pg=7.312311 kl=0.000000 reward=1.000000 skipped=462 +[train] epoch=8 step=464 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=463 +[train] epoch=8 step=465 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=464 +[train] epoch=8 step=466 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=465 +[train] epoch=8 step=467 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=466 +[train] epoch=8 step=468 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=467 +[train] epoch=8 step=469 loss=5.792642 pg=5.792642 kl=0.000000 reward=1.000000 skipped=468 +[train] epoch=8 step=470 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=469 +[train] epoch=8 step=471 loss=8.245316 pg=8.245316 kl=0.000000 reward=1.000000 skipped=470 +[train] epoch=8 step=472 loss=6.746428 pg=6.746428 kl=0.000000 reward=1.000000 skipped=471 +[train] epoch=8 step=473 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=472 +[train] epoch=8 step=474 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=473 +[train] epoch=8 step=475 loss=0.000013 pg=0.000000 kl=0.001251 reward=0.000000 skipped=474 +[train] epoch=8 step=476 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=475 +[train] epoch=8 step=477 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=476 +[train] epoch=8 step=478 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=477 +[train] epoch=8 step=479 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=478 +[train] epoch=8 step=480 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=479 +[train] epoch=8 step=481 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=480 +[train] epoch=8 step=482 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=481 +[train] epoch=8 step=483 loss=5.568174 pg=5.568095 kl=0.007986 reward=1.000000 skipped=482 +[train] epoch=8 step=484 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=483 +[train] epoch=8 step=485 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=484 +[train] epoch=8 step=486 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=485 +[train] epoch=8 step=487 loss=5.581098 pg=5.581098 kl=0.000000 reward=1.000000 skipped=486 +[train] epoch=8 step=488 loss=5.581098 pg=5.581098 kl=0.000000 reward=1.000000 skipped=487 +[train] epoch=8 step=489 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=488 +[train] epoch=8 step=490 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=489 +[train] epoch=8 step=491 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=490 +[train] epoch=8 step=492 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=491 +[train] epoch=8 step=493 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=492 +[train] epoch=8 step=494 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=493 +[train] epoch=8 step=495 loss=7.204127 pg=7.204088 kl=0.003951 reward=1.000000 skipped=494 +[train] epoch=8 step=496 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=495 +[train] epoch=8 step=497 loss=-6.001044 pg=-6.001044 kl=0.000000 reward=-1.000000 skipped=496 +[train] epoch=8 step=498 loss=-11.342767 pg=-11.342767 kl=0.000000 reward=-1.732051 skipped=497 +[train] epoch=8 step=499 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=498 +[train] epoch=8 step=500 loss=7.314037 pg=7.314037 kl=0.000000 reward=1.000000 skipped=499 +[train] epoch=8 step=501 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=500 +[train] epoch=8 step=502 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=501 +[train] epoch=8 step=503 loss=5.875602 pg=5.875589 kl=0.001239 reward=1.000000 skipped=502 +[train] epoch=8 step=504 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=503 +[train] epoch=8 step=505 loss=-6.788513 pg=-6.788513 kl=0.000000 reward=-1.000000 skipped=504 +[train] epoch=8 step=506 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=505 +[train] epoch=8 step=507 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=506 +[train] epoch=8 step=508 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=507 +[train] epoch=8 step=509 loss=5.282966 pg=5.282931 kl=0.003500 reward=1.000000 skipped=508 +[train] epoch=8 step=510 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=509 +[train] epoch=8 step=511 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=510 +[train] epoch=8 step=512 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=511 +[train] epoch=8 checkpoint_saved=/data4/runqing/dLLM-RL/rl_llava_next_video_reversal/ckpt/optimized diff --git a/wandb/run-20260601_133700-9x12ylug/files/requirements.txt b/wandb/run-20260601_133700-9x12ylug/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..75f723721d959cc9661b3ed252c3052267b34822 --- /dev/null +++ b/wandb/run-20260601_133700-9x12ylug/files/requirements.txt @@ -0,0 +1,167 @@ +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +tokenizers==0.15.2 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +markdown-it-py==4.2.0 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +accelerate==1.13.0 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +six==1.17.0 +Pygments==2.19.2 +open_clip_torch==3.3.0 +transformers==4.40.0.dev0 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +mdurl==0.1.2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +rich==15.0.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +setuptools==80.10.2 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +peft==0.4.0 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +aiosignal==1.4.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +shtab==1.8.0 +opencv-python-headless==4.9.0.80 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +docstring_parser==0.18.0 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +tyro==0.8.3 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +ftfy==6.3.1 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +timm==1.0.27 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260601_133700-9x12ylug/files/wandb-metadata.json b/wandb/run-20260601_133700-9x12ylug/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ce54d8ac9faafe60ad0cc54b62157e5333ea19d9 --- /dev/null +++ b/wandb/run-20260601_133700-9x12ylug/files/wandb-metadata.json @@ -0,0 +1,94 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-01T05:37:00.841096Z", + "args": [ + "config=configs/rl_llava_next_video_reversal.yaml", + "wandb.run_id=9x12ylug", + "experiment.current_epoch=8" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_llava_next_video.py", + "codePath": "train/rl_llava_next_video.py", + "codePathLocal": "train/rl_llava_next_video.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/llava-next-video/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "626052399104" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "f2it9tpacxjug20nb037ehmm1thy422p" +} \ No newline at end of file diff --git a/wandb/run-20260601_133700-9x12ylug/files/wandb-summary.json b/wandb/run-20260601_133700-9x12ylug/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7c86da7f512c3efe0c13da02dc74a2740c6aaef0 --- /dev/null +++ b/wandb/run-20260601_133700-9x12ylug/files/wandb-summary.json @@ -0,0 +1 @@ +{"_timestamp":1.7802927750649087e+09,"_wandb":{"runtime":4665},"train/skipped_steps":511,"train/advantage":1,"train/loss":8.03286361694336,"_runtime":4665,"_step":4095,"train/step":512,"train/pg_loss":8.03286361694336,"train/kl_loss":0} \ No newline at end of file diff --git a/wandb/run-20260601_133700-9x12ylug/logs/debug-core.log b/wandb/run-20260601_133700-9x12ylug/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..c64072de91b0846ab99d7502d8363b0b7ba1bf12 --- /dev/null +++ b/wandb/run-20260601_133700-9x12ylug/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-06-01T13:37:00.908334673+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpvirf6pbb/port-2057722.txt","pid":2057722,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-01T13:37:00.910050372+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2057722} +{"time":"2026-06-01T13:37:00.910040078+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2057722-2057953-2041219008/socket","Net":"unix"}} +{"time":"2026-06-01T13:37:01.088487089+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-01T13:37:01.098610655+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T13:37:02.098403799+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T13:37:07.444431359+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"blgackwk0qsg"} +{"time":"2026-06-01T13:46:53.111451313+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"blgackwk0qsg"} +{"time":"2026-06-01T13:46:53.349012202+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"blgackwk0qsg"} +{"time":"2026-06-01T13:46:53.349409868+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T13:46:53.35058331+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T13:46:53.36123846+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-01T13:46:53.361262365+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-01T13:46:53.361277025+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-01T13:46:53.361328984+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-01T13:46:53.361378249+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2057722-2057953-2041219008/socket","Net":"unix"}} +{"time":"2026-06-01T13:46:53.361425776+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-01T13:46:53.361442811+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-01T13:46:53.361458056+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260601_133700-9x12ylug/logs/debug-internal.log b/wandb/run-20260601_133700-9x12ylug/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3bb3420e705078b4ef624a643ab549a37046b9e5 --- /dev/null +++ b/wandb/run-20260601_133700-9x12ylug/logs/debug-internal.log @@ -0,0 +1,30 @@ +{"time":"2026-06-01T13:37:01.098752461+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-01T13:37:02.097922521+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-01T13:37:02.09806869+08:00","level":"INFO","msg":"stream: created new stream","id":"9x12ylug"} +{"time":"2026-06-01T13:37:02.098150107+08:00","level":"INFO","msg":"handler: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T13:37:02.098384807+08:00","level":"INFO","msg":"stream: started","id":"9x12ylug"} +{"time":"2026-06-01T13:37:02.098426422+08:00","level":"INFO","msg":"sender: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T13:37:02.098451719+08:00","level":"INFO","msg":"writer: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T13:37:02.102519016+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-01T13:37:07.401003956+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T053707Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c5e635b86730e28357ed9bd5b9e0f08a763e37da082262873e73c742dab1064c"} +{"time":"2026-06-01T13:37:07.401166235+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json18B4DE5930E6F61829678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_133700-9x12ylug/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T053707Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c5e635b86730e28357ed9bd5b9e0f08a763e37da082262873e73c742dab1064c, Size: 2703}"} +{"time":"2026-06-01T13:37:07.401814532+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T053707Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5afab7c8036fe95320a95db909ada764dafd61848404c45336731941656445c9"} +{"time":"2026-06-01T13:37:07.40194101+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txtnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt18B4DE593184935F17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_133700-9x12ylug/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T053707Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5afab7c8036fe95320a95db909ada764dafd61848404c45336731941656445c9, Size: 3167}"} +{"time":"2026-06-01T13:37:07.405415754+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T053707Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5d41bb44aef939a7e78ec18f82a1855fc34510b0e11c32b5698c434ec465e124"} +{"time":"2026-06-01T13:37:07.405576153+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.pynetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py18B4DE593178667317e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_133700-9x12ylug/files/code/train/rl_llava_next_video.py, Name: code/train/rl_llava_next_video.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T053707Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5d41bb44aef939a7e78ec18f82a1855fc34510b0e11c32b5698c434ec465e124, Size: 15437}"} +{"time":"2026-06-01T13:37:07.407220229+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T053707Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=fd6fe8b640d27a2c416afb124a369492ceaba9d11f3b2ac13eda1b1e936d6a4b"} +{"time":"2026-06-01T13:37:07.407325223+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch18B4DE59317E59BB17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_133700-9x12ylug/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T053707Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=fd6fe8b640d27a2c416afb124a369492ceaba9d11f3b2ac13eda1b1e936d6a4b, Size: 14311}"} +{"time":"2026-06-01T13:37:07.409106073+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T053707Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=50b516916fa7472c486e4395f4e41222d33ccd61d0db7c2d470d553332e95578"} +{"time":"2026-06-01T13:37:07.409195436+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18B4DE593194D2BB17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_133700-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T053707Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=50b516916fa7472c486e4395f4e41222d33ccd61d0db7c2d470d553332e95578, Size: 14311}"} +{"time":"2026-06-01T13:46:53.286367333+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T054653Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b6eb8874e24581c600696a75813c43a126ead67e10897d926760b5e3499a7ea5"} +{"time":"2026-06-01T13:46:53.286539189+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json18B4DEE19AFD5D9117e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_133700-9x12ylug/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T054653Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b6eb8874e24581c600696a75813c43a126ead67e10897d926760b5e3499a7ea5, Size: 236}"} +{"time":"2026-06-01T13:46:53.287245523+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T054653Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7366b69fa8db4fff70d368a49d760c2190ae3c274735d9ed57837855b1054ea4"} +{"time":"2026-06-01T13:46:53.287350154+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yamlnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml18B4DEE19A63513A29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_133700-9x12ylug/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T054653Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7366b69fa8db4fff70d368a49d760c2190ae3c274735d9ed57837855b1054ea4, Size: 7147}"} +{"time":"2026-06-01T13:46:53.303138938+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T054653Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b627fb8a5162f8994f68931bcbfa3f0b635ea123127abd87c5b1750fdecc3080"} +{"time":"2026-06-01T13:46:53.303259933+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.lognetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log18B4DEE19B01C80917e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_133700-9x12ylug/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T054653Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b627fb8a5162f8994f68931bcbfa3f0b635ea123127abd87c5b1750fdecc3080, Size: 48362}"} +{"time":"2026-06-01T13:46:53.303997268+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-01T13:46:53.344522779+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-01T13:46:53.349427631+08:00","level":"INFO","msg":"stream: closing","id":"9x12ylug"} +{"time":"2026-06-01T13:46:53.349441207+08:00","level":"INFO","msg":"handler: closed","stream_id":"9x12ylug"} +{"time":"2026-06-01T13:46:53.349509634+08:00","level":"INFO","msg":"sender: closed","stream_id":"9x12ylug"} +{"time":"2026-06-01T13:46:53.349525079+08:00","level":"INFO","msg":"stream: closed","id":"9x12ylug"} diff --git a/wandb/run-20260601_133700-9x12ylug/logs/debug.log b/wandb/run-20260601_133700-9x12ylug/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4e6552a26eff581de6b96d700b28a885da1e0f52 --- /dev/null +++ b/wandb/run-20260601_133700-9x12ylug/logs/debug.log @@ -0,0 +1,25 @@ +2026-06-01 13:37:00,843 INFO MainThread:2057722 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-01 13:37:00,843 INFO MainThread:2057722 [wandb_setup.py:_flush():81] Configure stats pid to 2057722 +2026-06-01 13:37:00,843 INFO MainThread:2057722 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-01 13:37:00,844 INFO MainThread:2057722 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260601_133700-9x12ylug/logs/debug.log +2026-06-01 13:37:00,844 INFO MainThread:2057722 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260601_133700-9x12ylug/logs/debug-internal.log +2026-06-01 13:37:00,844 INFO MainThread:2057722 [wandb_init.py:init():844] calling init triggers +2026-06-01 13:37:00,844 INFO MainThread:2057722 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': '9x12ylug', 'system.env_name': '/data8/runqing/miniconda/envs/llava-next-video', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_llava_next_video_reversal', 'experiment.function': 'train', 'experiment.current_epoch': 8, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.reference_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_name': 'llava_llama', 'model.model_base': None, 'model.conv_mode': 'vicuna_v1', 'model.load_overwrite_config.mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'model.load_overwrite_config.mm_spatial_pool_stride': 2, 'model.load_overwrite_config.mm_spatial_pool_mode': 'average', 'model.load_overwrite_config.mm_newline_position': 'no_token', 'model.load_overwrite_config.max_sequence_length': 8192, 'model.load_overwrite_config.tokenizer_model_max_length': 8192, 'model.load_overwrite_config.rope_scaling.factor': 2.0, 'model.load_overwrite_config.rope_scaling.type': 'linear', 'model.load_overwrite_config.video_fps': 1, 'model.load_overwrite_config.frames_upbound': 32, 'model.load_overwrite_config.force_sample': True, 'model.load_overwrite_config.add_time_instruction': True, 'dataset.train_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_train.json', 'dataset.optimization_data_file': 'rl_data.json', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.top_p': 0.95, 'rollout.max_new_tokens': 128, 'reward.max_direction_gap': 0.2, 'reward.direction_consistency_bonus': 0.3, 'training.num_train_epochs': 1, 'training.gradient_accumulation_steps': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.max_grad_norm': 1.0, 'training.kl_beta': 0.01, 'training.logging_steps': 1, 'training.trainable_parts.0': 'mm_projector', 'training.load_overwrite_config.max_sequence_length': 2048, 'training.load_overwrite_config.tokenizer_model_max_length': 2048, 'training.load_overwrite_config.frames_upbound': 8, 'optimizer.learning_rate': 1e-06, 'optimizer.weight_decay': 0.0, 'evaluation.eval_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json', 'evaluation.num_task_per_step': 64, 'evaluation.num_response_per_task': 3, 'evaluation.max_new_tokens': 128, 'evaluation.temperature': 0.1, 'config': 'configs/rl_llava_next_video_reversal.yaml', '_wandb': {'code_path': 'code/train/rl_llava_next_video.py'}} +2026-06-01 13:37:00,844 INFO MainThread:2057722 [wandb_init.py:init():892] starting backend +2026-06-01 13:37:01,089 INFO MainThread:2057722 [wandb_init.py:init():895] sending inform_init request +2026-06-01 13:37:01,094 INFO MainThread:2057722 [wandb_init.py:init():903] backend started and connected +2026-06-01 13:37:01,095 INFO MainThread:2057722 [wandb_init.py:init():973] updated telemetry +2026-06-01 13:37:01,098 INFO MainThread:2057722 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-01 13:37:02,324 INFO MainThread:2057722 [wandb_init.py:init():1037] run resumed +2026-06-01 13:37:02,328 INFO MainThread:2057722 [wandb_init.py:init():1042] starting run threads in backend +2026-06-01 13:37:02,440 INFO MainThread:2057722 [wandb_run.py:_console_start():2524] atexit reg +2026-06-01 13:37:02,440 INFO MainThread:2057722 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-01 13:37:02,440 INFO MainThread:2057722 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-01 13:37:02,440 INFO MainThread:2057722 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-01 13:37:02,444 INFO MainThread:2057722 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-01 13:46:53,110 INFO MainThread:2057722 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_llava_next_video_reversal/9x12ylug +2026-06-01 13:46:53,110 INFO MainThread:2057722 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-01 13:46:53,111 INFO MainThread:2057722 [wandb_run.py:_restore():2472] restore +2026-06-01 13:46:53,111 INFO MainThread:2057722 [wandb_run.py:_restore():2478] restore done +2026-06-01 13:46:53,348 INFO MainThread:2057722 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260601_153643-9x12ylug/files/code/train/rl_llava_next_video.py b/wandb/run-20260601_153643-9x12ylug/files/code/train/rl_llava_next_video.py new file mode 100644 index 0000000000000000000000000000000000000000..8199163c2bde56855ea46bec0831fe434956172b --- /dev/null +++ b/wandb/run-20260601_153643-9x12ylug/files/code/train/rl_llava_next_video.py @@ -0,0 +1,358 @@ +import copy +import os +import sys +from typing import Iterable, List +import warnings +import random + +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import DataLoader, Dataset +import wandb +from accelerate import Accelerator +from omegaconf import OmegaConf + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LLAVA_NEXT_ROOT = os.path.join(REPO_ROOT, "LLaVA-NeXT") +sys.path.insert(0, LLAVA_NEXT_ROOT) +sys.path.insert(1, REPO_ROOT) + +from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX +from llava.conversation import conv_templates +from llava.mm_utils import tokenizer_image_token +from llava.model.builder import load_pretrained_model +from llava.utils import disable_torch_init, process_video_with_decord + +from train.llava_next_video.common import ( + build_video_mcq_prompt, + ensure_dir, + get_config, + load_json, + resolve_media_abs_path, +) +from train.utils import ensure_wandb_run_id, flatten_omega_conf + +warnings.filterwarnings("ignore", message=".*copying from a non-meta parameter.*") +warnings.filterwarnings("ignore", message=".*Flash Attention 2.0 with a model not initialized on GPU.*") + + +class RLDataset(Dataset): + def __init__(self, samples): + self.samples = samples + + def __len__(self): + return len(self.samples) + + def __getitem__(self, idx): + return self.samples[idx] + + +def build_prompt_text(sample: dict) -> str: + return build_video_mcq_prompt(sample) + + +def tokenize_example(tokenizer, conv_mode: str, prompt_text: str, response_text: str): + prompt_conv = copy.deepcopy(conv_templates[conv_mode]) + prompt_conv.append_message(prompt_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + prompt_conv.append_message(prompt_conv.roles[1], None) + prompt_ids = tokenizer_image_token(prompt_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + full_conv = copy.deepcopy(conv_templates[conv_mode]) + full_conv.append_message(full_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + full_conv.append_message(full_conv.roles[1], response_text) + full_ids = tokenizer_image_token(full_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + labels = full_ids.clone() + labels[: prompt_ids.numel()] = -100 + return full_ids, labels + + +def load_video_inputs(sample: dict, image_processor, model_cfg, device): + video_path = resolve_media_abs_path(sample, None, media_type="video") + video, _, _, _ = process_video_with_decord(video_path, model_cfg) + pixel_values = image_processor.preprocess(video, return_tensors="pt")["pixel_values"].to(device=device, dtype=torch.float16) + return [pixel_values], [video[0].size], ["video"] + + +def compute_token_logps(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: + shifted_logits = logits[:, :-1, :] + shifted_labels = labels[:, 1:] + valid_mask = shifted_labels.ne(-100) + gather_labels = shifted_labels.masked_fill(~valid_mask, 0) + token_logps = F.log_softmax(shifted_logits, dim=-1).gather(-1, gather_labels.unsqueeze(-1)).squeeze(-1) + token_logps = token_logps[valid_mask] + return token_logps[torch.isfinite(token_logps)] + + +def safe_mean(values: torch.Tensor, *, device: torch.device, dtype: torch.dtype) -> torch.Tensor: + if values.numel() == 0: + return torch.zeros((), device=device, dtype=dtype) + return values.mean() + + +def snapshot_named_parameters(model, names: Iterable[str]) -> dict[str, torch.Tensor]: + state = {} + target_names = set(names) + for name, param in model.named_parameters(): + if name in target_names: + state[name] = param.detach().cpu().clone() + return state + + +def restore_named_parameters(model, state: dict[str, torch.Tensor]) -> None: + for name, param in model.named_parameters(): + if name in state: + param.data.copy_(state[name].to(device=param.device, dtype=param.dtype)) + + +def gradients_are_finite(params: Iterable[torch.nn.Parameter]) -> bool: + for param in params: + if param.grad is not None and not torch.isfinite(param.grad).all(): + return False + return True + + +def parameters_are_finite(params: Iterable[torch.nn.Parameter]) -> bool: + return all(torch.isfinite(param).all() for param in params) + + +def _merge_overwrite_config(config) -> dict: + overwrite_config = OmegaConf.to_container(config.model.get("load_overwrite_config", {}), resolve=True) or {} + training_overrides = OmegaConf.to_container(config.training.get("load_overwrite_config", {}), resolve=True) or {} + overwrite_config.update(training_overrides) + return overwrite_config + + +def _set_trainable_parameters(model, trainable_parts: Iterable[str]) -> List[torch.nn.Parameter]: + part_to_patterns = { + "lm_head": ["lm_head"], + "embed_tokens": ["embed_tokens"], + "mm_projector": ["mm_projector"], + "vision_resampler": ["vision_resampler"], + "vision_tower": ["vision_tower"], + } + enabled_parts = list(trainable_parts) + enabled_patterns = [] + for part in enabled_parts: + if part == "all": + for param in model.parameters(): + param.requires_grad_(True) + return list(model.parameters()) + enabled_patterns.extend(part_to_patterns.get(part, [part])) + + trainable_params: List[torch.nn.Parameter] = [] + seen_param_ids = set() + for name, param in model.named_parameters(): + should_train = any(pattern in name for pattern in enabled_patterns) + param.requires_grad_(should_train) + if should_train and id(param) not in seen_param_ids: + trainable_params.append(param) + seen_param_ids.add(id(param)) + return trainable_params + + +def _summarize_trainable_parameters(model) -> tuple[int, int, List[str]]: + total_params = 0 + trainable_params = 0 + trainable_names = [] + for name, param in model.named_parameters(): + count = param.numel() + total_params += count + if param.requires_grad: + trainable_params += count + trainable_names.append(name) + return trainable_params, total_params, trainable_names + + +def set_seed(seed: int) -> None: + random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + + +if __name__ == "__main__": + config = get_config() + disable_torch_init() + set_seed(int(config.training.get("seed", 10086))) + if hasattr(torch.backends, "cuda") and torch.cuda.is_available(): + torch.backends.cuda.matmul.allow_tf32 = bool(config.training.get("enable_tf32", True)) + if hasattr(torch.backends, "cudnn"): + torch.backends.cudnn.allow_tf32 = bool(config.training.get("enable_tf32", True)) + + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + ) + + existing_run_id = ensure_wandb_run_id(config) + if accelerator.is_main_process: + wandb.init( + project=config.experiment.project, + id=existing_run_id, + resume=config.wandb.resume, + config={k: v for k, v in flatten_omega_conf(config, resolve=True)}, + name=config.experiment.project, + reinit=True, + ) + + current_epoch = int(config.experiment.current_epoch) + model_path = config.model.pretrained_model if current_epoch == 1 else os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ref_model_path = config.model.reference_model or config.model.pretrained_model + train_samples = load_json(os.path.join(config.experiment.project, "temp_data", config.dataset.optimization_data_file)) + overwrite_config = _merge_overwrite_config(config) + ref_device = torch.device("cuda:1") if torch.cuda.is_available() and torch.cuda.device_count() > 1 else accelerator.device + if accelerator.is_main_process: + print( + f"[train] epoch={current_epoch} samples={len(train_samples)} " + f"grad_accum={int(config.training.gradient_accumulation_steps)}", + flush=True, + ) + + tokenizer, model, image_processor, _ = load_pretrained_model( + model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + _, ref_model, _, _ = load_pretrained_model( + ref_model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + ref_model.to(device=ref_device, dtype=torch.float16) + ref_model.eval() + for p in ref_model.parameters(): + p.requires_grad_(False) + + trainable_parts = list(config.training.get("trainable_parts", ["lm_head"])) + trainable_params = _set_trainable_parameters(model, trainable_parts) + if not trainable_params: + raise ValueError(f"No trainable parameters matched training.trainable_parts={trainable_parts}") + + trainable_count, total_count, trainable_names = _summarize_trainable_parameters(model) + if accelerator.is_main_process: + print( + f"RL trainable parts: {trainable_parts}; " + f"trainable params={trainable_count:,}/{total_count:,} " + f"({100.0 * trainable_count / max(total_count, 1):.4f}%)" + ) + for name in trainable_names[:20]: + print(f" trainable: {name}") + if len(trainable_names) > 20: + print(f" ... and {len(trainable_names) - 20} more") + + optimizer = AdamW(trainable_params, lr=float(config.optimizer.learning_rate), weight_decay=float(config.optimizer.weight_decay)) + dataloader = DataLoader(RLDataset(train_samples), batch_size=1, shuffle=True) + + model, optimizer, dataloader = accelerator.prepare(model, optimizer, dataloader) + last_good_state = snapshot_named_parameters(accelerator.unwrap_model(model), trainable_names) + model.train() + + global_step = 0 + skipped_steps = 0 + for _ in range(int(config.training.num_train_epochs)): + for batch in dataloader: + sample = {k: v[0] if isinstance(v, list) else v for k, v in batch.items()} + prompt_text = build_prompt_text(sample) + input_ids, labels = tokenize_example(tokenizer, config.model.conv_mode, prompt_text, sample["response"]) + input_ids = input_ids.unsqueeze(0).to(accelerator.device) + labels = labels.unsqueeze(0).to(accelerator.device) + attention_mask = input_ids.ne(tokenizer.pad_token_id if tokenizer.pad_token_id is not None else 0).long() + images, image_sizes, modalities = load_video_inputs(sample, image_processor, model.config, accelerator.device) + ref_input_ids = input_ids.to(ref_device) + ref_attention_mask = attention_mask.to(ref_device) + ref_images = [img.to(ref_device) for img in images] + ref_labels = labels.to(ref_device) + + with accelerator.accumulate(model): + outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + images=images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + with torch.no_grad(): + ref_outputs = ref_model( + input_ids=ref_input_ids, + attention_mask=ref_attention_mask, + images=ref_images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + + policy_logps = compute_token_logps(outputs.logits, labels) + ref_logps = compute_token_logps(ref_outputs.logits, ref_labels).to(policy_logps.device) + shared_tokens = min(policy_logps.numel(), ref_logps.numel()) + if shared_tokens == 0: + skipped_steps += 1 + optimizer.zero_grad() + continue + policy_logps = policy_logps[:shared_tokens] + ref_logps = ref_logps[:shared_tokens] + advantage = torch.tensor(float(sample["reward"]), device=accelerator.device, dtype=policy_logps.dtype) + pg_loss = -(advantage * safe_mean(policy_logps, device=accelerator.device, dtype=policy_logps.dtype)) + kl_loss = torch.clamp( + safe_mean(policy_logps - ref_logps, device=accelerator.device, dtype=policy_logps.dtype), + min=0.0, + ) + loss = pg_loss + float(config.training.kl_beta) * kl_loss + if not torch.isfinite(loss): + skipped_steps += 1 + optimizer.zero_grad() + continue + + accelerator.backward(loss) + if not gradients_are_finite(trainable_params): + skipped_steps += 1 + optimizer.zero_grad() + restore_named_parameters(accelerator.unwrap_model(model), last_good_state) + continue + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, float(config.training.max_grad_norm)) + optimizer.step() + if not parameters_are_finite(trainable_params): + skipped_steps += 1 + restore_named_parameters(accelerator.unwrap_model(model), last_good_state) + optimizer.zero_grad() + continue + last_good_state = snapshot_named_parameters(accelerator.unwrap_model(model), trainable_names) + optimizer.zero_grad() + + global_step += 1 + if accelerator.is_main_process and global_step % int(config.training.logging_steps) == 0: + print( + f"[train] epoch={current_epoch} step={global_step} " + f"loss={float(loss.detach().cpu()):.6f} " + f"pg={float(pg_loss.detach().cpu()):.6f} " + f"kl={float(kl_loss.detach().cpu()):.6f} " + f"reward={float(advantage.detach().cpu()):.6f} " + f"skipped={skipped_steps}", + flush=True, + ) + wandb.log( + { + "train/loss": float(loss.detach().cpu()), + "train/pg_loss": float(pg_loss.detach().cpu()), + "train/kl_loss": float(kl_loss.detach().cpu()), + "train/advantage": float(advantage.detach().cpu()), + "train/skipped_steps": skipped_steps, + "train/step": global_step, + } + ) + + accelerator.wait_for_everyone() + if accelerator.is_main_process: + out_dir = os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ensure_dir(out_dir) + accelerator.unwrap_model(model).save_pretrained(out_dir, safe_serialization=True) + tokenizer.save_pretrained(out_dir) + print(f"[train] epoch={current_epoch} checkpoint_saved={os.path.abspath(out_dir)}", flush=True) + wandb.finish() diff --git a/wandb/run-20260601_153643-9x12ylug/files/config.yaml b/wandb/run-20260601_153643-9x12ylug/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1961ac889971ebd5e154abe0025f3c0006a11bc --- /dev/null +++ b/wandb/run-20260601_153643-9x12ylug/files/config.yaml @@ -0,0 +1,223 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_llava_next_video.py + e: + 0m4rjmb3ixcch54jq46xifhzvmedpduo: + args: + - config=configs/rl_llava_next_video_reversal.yaml + - wandb.run_id=9x12ylug + - experiment.current_epoch=16 + codePath: train/rl_llava_next_video.py + codePathLocal: train/rl_llava_next_video.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "626064564224" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/llava-next-video/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_llava_next_video.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-06-01T07:36:43.643861Z" + writerId: 0m4rjmb3ixcch54jq46xifhzvmedpduo + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + "2": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.40.0.dev0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_llava_next_video_reversal.yaml +dataset.optimization_data_file: + value: rl_data.json +dataset.train_json: + value: /data4/runqing/dLLM-RL/data/video_reversal_rl_train.json +evaluation.eval_json: + value: /data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json +evaluation.max_new_tokens: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.num_task_per_step: + value: 64 +evaluation.temperature: + value: 0.1 +experiment.current_epoch: + value: 16 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.project: + value: rl_llava_next_video_reversal +experiment.save_every: + value: 10 +experiment.total_step: + value: 30 +model.conv_mode: + value: vicuna_v1 +model.load_overwrite_config.add_time_instruction: + value: true +model.load_overwrite_config.force_sample: + value: true +model.load_overwrite_config.frames_upbound: + value: 32 +model.load_overwrite_config.max_sequence_length: + value: 8192 +model.load_overwrite_config.mm_newline_position: + value: no_token +model.load_overwrite_config.mm_spatial_pool_mode: + value: average +model.load_overwrite_config.mm_spatial_pool_stride: + value: 2 +model.load_overwrite_config.mm_vision_tower: + value: /data4/runqing/hf/clip-vit-large-patch14-336 +model.load_overwrite_config.rope_scaling.factor: + value: 2 +model.load_overwrite_config.rope_scaling.type: + value: linear +model.load_overwrite_config.tokenizer_model_max_length: + value: 8192 +model.load_overwrite_config.video_fps: + value: 1 +model.model_base: + value: null +model.model_name: + value: llava_llama +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +model.reference_model: + value: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +optimizer.learning_rate: + value: 1e-06 +optimizer.weight_decay: + value: 0 +reward.direction_consistency_bonus: + value: 0.3 +reward.max_direction_gap: + value: 0.2 +rollout.max_new_tokens: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.temperature: + value: 0.8 +rollout.top_p: + value: 0.95 +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/llava-next-video +training.enable_tf32: + value: true +training.gradient_accumulation_steps: + value: 2 +training.kl_beta: + value: 0.01 +training.load_overwrite_config.frames_upbound: + value: 8 +training.load_overwrite_config.max_sequence_length: + value: 2048 +training.load_overwrite_config.tokenizer_model_max_length: + value: 2048 +training.logging_steps: + value: 1 +training.max_grad_norm: + value: 1 +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.seed: + value: 10086 +training.trainable_parts.0: + value: mm_projector +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: 9x12ylug diff --git a/wandb/run-20260601_153643-9x12ylug/files/diff.patch b/wandb/run-20260601_153643-9x12ylug/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260601_153643-9x12ylug/files/diff.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260601_153643-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260601_153643-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260601_153643-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260601_153643-9x12ylug/files/output.log b/wandb/run-20260601_153643-9x12ylug/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..bb73145e2ba40576decebf2391735f49ff3bd3e6 --- /dev/null +++ b/wandb/run-20260601_153643-9x12ylug/files/output.log @@ -0,0 +1,529 @@ +[train] epoch=16 samples=1024 grad_accum=2 +Loaded LLaVA model: rl_llava_next_video_reversal/ckpt/optimized +Overwriting config with {'mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'mm_spatial_pool_stride': 2, 'mm_spatial_pool_mode': 'average', 'mm_newline_position': 'no_token', 'max_sequence_length': 2048, 'tokenizer_model_max_length': 2048, 'rope_scaling': {'factor': 2.0, 'type': 'linear'}, 'video_fps': 1, 'frames_upbound': 8, 'force_sample': True, 'add_time_instruction': True} +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +The checkpoint seems to contain `vision_tower` weights: `mm_tunable_parts` contains `mm_vision_tower`. +Model Class: LlavaLlamaForCausalLM +Loaded LLaVA model: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +You are using a model of type llava to instantiate a model of type llava_llama. This is not supported for all configurations of models and can yield errors. +Overwriting config with {'mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'mm_spatial_pool_stride': 2, 'mm_spatial_pool_mode': 'average', 'mm_newline_position': 'no_token', 'max_sequence_length': 2048, 'tokenizer_model_max_length': 2048, 'rope_scaling': {'factor': 2.0, 'type': 'linear'}, 'video_fps': 1, 'frames_upbound': 8, 'force_sample': True, 'add_time_instruction': True} +The checkpoint seems to contain `vision_tower` weights: `mm_tunable_parts` contains `mm_vision_tower`. +Model Class: LlavaLlamaForCausalLM +RL trainable parts: ['mm_projector']; trainable params=20,979,712/7,062,915,072 (0.2970%) + trainable: model.mm_projector.0.weight + trainable: model.mm_projector.0.bias + trainable: model.mm_projector.2.weight + trainable: model.mm_projector.2.bias +[train] epoch=16 step=1 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=0 +[train] epoch=16 step=2 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=1 +[train] epoch=16 step=3 loss=6.282383 pg=6.282336 kl=0.004788 reward=1.000000 skipped=2 +[train] epoch=16 step=4 loss=6.468365 pg=6.468350 kl=0.001409 reward=1.000000 skipped=3 +[train] epoch=16 step=5 loss=7.083083 pg=7.083083 kl=0.000000 reward=1.000000 skipped=4 +[train] epoch=16 step=6 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=5 +[train] epoch=16 step=7 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=6 +[train] epoch=16 step=8 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=7 +[train] epoch=16 step=9 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=8 +[train] epoch=16 step=10 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=9 +[train] epoch=16 step=11 loss=3.049087 pg=3.049040 kl=0.004637 reward=0.577350 skipped=10 +[train] epoch=16 step=12 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=11 +[train] epoch=16 step=13 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=12 +[train] epoch=16 step=14 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=13 +[train] epoch=16 step=15 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=14 +[train] epoch=16 step=16 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=15 +[train] epoch=16 step=17 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=16 +[train] epoch=16 step=18 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=17 +[train] epoch=16 step=19 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=18 +[train] epoch=16 step=20 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=19 +[train] epoch=16 step=21 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=20 +[train] epoch=16 step=22 loss=7.845365 pg=7.845365 kl=0.000000 reward=1.000000 skipped=21 +[train] epoch=16 step=23 loss=7.083083 pg=7.083083 kl=0.000000 reward=1.000000 skipped=22 +[train] epoch=16 step=24 loss=5.951652 pg=5.951652 kl=0.000000 reward=1.000000 skipped=23 +[train] epoch=16 step=25 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=24 +[train] epoch=16 step=26 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=25 +[train] epoch=16 step=27 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=26 +[train] epoch=16 step=28 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=27 +[train] epoch=16 step=29 loss=4.836777 pg=4.836777 kl=0.000000 reward=1.000000 skipped=28 +[train] epoch=16 step=30 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=29 +[train] epoch=16 step=31 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=30 +[train] epoch=16 step=32 loss=5.878745 pg=5.878745 kl=0.000000 reward=1.000000 skipped=31 +[train] epoch=16 step=33 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=32 +[train] epoch=16 step=34 loss=7.771910 pg=7.771876 kl=0.003357 reward=1.000000 skipped=33 +[train] epoch=16 step=35 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=34 +[train] epoch=16 step=36 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=35 +[train] epoch=16 step=37 loss=8.125317 pg=8.125317 kl=0.000000 reward=1.000000 skipped=36 +[train] epoch=16 step=38 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=37 +[train] epoch=16 step=39 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=38 +[train] epoch=16 step=40 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=39 +[train] epoch=16 step=41 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=40 +[train] epoch=16 step=42 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=41 +[train] epoch=16 step=43 loss=3.049087 pg=3.049040 kl=0.004637 reward=0.577350 skipped=42 +[train] epoch=16 step=44 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=43 +[train] epoch=16 step=45 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=44 +[train] epoch=16 step=46 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=45 +[train] epoch=16 step=47 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=46 +[train] epoch=16 step=48 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=47 +[train] epoch=16 step=49 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=48 +[train] epoch=16 step=50 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=49 +[train] epoch=16 step=51 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=50 +[train] epoch=16 step=52 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=51 +[train] epoch=16 step=53 loss=-2.338539 pg=-2.338539 kl=0.000000 reward=-0.377964 skipped=52 +[train] epoch=16 step=54 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=53 +[train] epoch=16 step=55 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=54 +[train] epoch=16 step=56 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=55 +[train] epoch=16 step=57 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=56 +[train] epoch=16 step=58 loss=5.665429 pg=5.665429 kl=0.000000 reward=0.774597 skipped=57 +[train] epoch=16 step=59 loss=5.369410 pg=5.369405 kl=0.000436 reward=1.000000 skipped=58 +[train] epoch=16 step=60 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=59 +[train] epoch=16 step=61 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=60 +[train] epoch=16 step=62 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=61 +[train] epoch=16 step=63 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=62 +[train] epoch=16 step=64 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=63 +[train] epoch=16 step=65 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=64 +[train] epoch=16 step=66 loss=4.211183 pg=4.211183 kl=0.000000 reward=0.774597 skipped=65 +[train] epoch=16 step=67 loss=5.568174 pg=5.568095 kl=0.007986 reward=1.000000 skipped=66 +[train] epoch=16 step=68 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=67 +[train] epoch=16 step=69 loss=6.862856 pg=6.862832 kl=0.002382 reward=1.000000 skipped=68 +[train] epoch=16 step=70 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=69 +[train] epoch=16 step=71 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=70 +[train] epoch=16 step=72 loss=6.468365 pg=6.468350 kl=0.001409 reward=1.000000 skipped=71 +[train] epoch=16 step=73 loss=8.117537 pg=8.117537 kl=0.000057 reward=1.000000 skipped=72 +[train] epoch=16 step=74 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=73 +[train] epoch=16 step=75 loss=8.125317 pg=8.125317 kl=0.000000 reward=1.000000 skipped=74 +[train] epoch=16 step=76 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=75 +[train] epoch=16 step=77 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=76 +[train] epoch=16 step=78 loss=5.282966 pg=5.282931 kl=0.003500 reward=1.000000 skipped=77 +[train] epoch=16 step=79 loss=5.875602 pg=5.875589 kl=0.001239 reward=1.000000 skipped=78 +[train] epoch=16 step=80 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=79 +[train] epoch=16 step=81 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=80 +[train] epoch=16 step=82 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=81 +[train] epoch=16 step=83 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=82 +[train] epoch=16 step=84 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=83 +[train] epoch=16 step=85 loss=7.771910 pg=7.771876 kl=0.003357 reward=1.000000 skipped=84 +[train] epoch=16 step=86 loss=6.194232 pg=6.194232 kl=0.000000 reward=1.000000 skipped=85 +[train] epoch=16 step=87 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=86 +[train] epoch=16 step=88 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=87 +[train] epoch=16 step=89 loss=4.836777 pg=4.836777 kl=0.000000 reward=1.000000 skipped=88 +[train] epoch=16 step=90 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=89 +[train] epoch=16 step=91 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=90 +[train] epoch=16 step=92 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=91 +[train] epoch=16 step=93 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=92 +[train] epoch=16 step=94 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=93 +[train] epoch=16 step=95 loss=7.021185 pg=7.021152 kl=0.003361 reward=1.000000 skipped=94 +[train] epoch=16 step=96 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=95 +[train] epoch=16 step=97 loss=7.256117 pg=7.256107 kl=0.000982 reward=1.000000 skipped=96 +[train] epoch=16 step=98 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=97 +[train] epoch=16 step=99 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=98 +[train] epoch=16 step=100 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=99 +[train] epoch=16 step=101 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=100 +[train] epoch=16 step=102 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=101 +[train] epoch=16 step=103 loss=6.126596 pg=6.126578 kl=0.001824 reward=1.000000 skipped=102 +[train] epoch=16 step=104 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=103 +[train] epoch=16 step=105 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=104 +[train] epoch=16 step=106 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=105 +[train] epoch=16 step=107 loss=6.295836 pg=6.295836 kl=0.000000 reward=1.000000 skipped=106 +[train] epoch=16 step=108 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=107 +[train] epoch=16 step=109 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=108 +[train] epoch=16 step=110 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=109 +[train] epoch=16 step=111 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=110 +[train] epoch=16 step=112 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=111 +[train] epoch=16 step=113 loss=6.295836 pg=6.295836 kl=0.000000 reward=1.000000 skipped=112 +[train] epoch=16 step=114 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=113 +[train] epoch=16 step=115 loss=6.295836 pg=6.295836 kl=0.000000 reward=1.000000 skipped=114 +[train] epoch=16 step=116 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=115 +[train] epoch=16 step=117 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=116 +[train] epoch=16 step=118 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=117 +[train] epoch=16 step=119 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=118 +[train] epoch=16 step=120 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=119 +[train] epoch=16 step=121 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=120 +[train] epoch=16 step=122 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=121 +[train] epoch=16 step=123 loss=7.845365 pg=7.845365 kl=0.000000 reward=1.000000 skipped=122 +[train] epoch=16 step=124 loss=6.414127 pg=6.414127 kl=0.000000 reward=1.000000 skipped=123 +[train] epoch=16 step=125 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=124 +[train] epoch=16 step=126 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=125 +[train] epoch=16 step=127 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=126 +[train] epoch=16 step=128 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=127 +[train] epoch=16 step=129 loss=6.862856 pg=6.862832 kl=0.002382 reward=1.000000 skipped=128 +[train] epoch=16 step=130 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=129 +[train] epoch=16 step=131 loss=7.021185 pg=7.021152 kl=0.003361 reward=1.000000 skipped=130 +[train] epoch=16 step=132 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=131 +[train] epoch=16 step=133 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=132 +[train] epoch=16 step=134 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=133 +[train] epoch=16 step=135 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=134 +[train] epoch=16 step=136 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=135 +[train] epoch=16 step=137 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=136 +[train] epoch=16 step=138 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=137 +[train] epoch=16 step=139 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=138 +[train] epoch=16 step=140 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=139 +[train] epoch=16 step=141 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=140 +[train] epoch=16 step=142 loss=5.734613 pg=5.734613 kl=0.000000 reward=1.000000 skipped=141 +[train] epoch=16 step=143 loss=5.878745 pg=5.878745 kl=0.000000 reward=1.000000 skipped=142 +[train] epoch=16 step=144 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=143 +[train] epoch=16 step=145 loss=-2.338539 pg=-2.338539 kl=0.000000 reward=-0.377964 skipped=144 +[train] epoch=16 step=146 loss=6.068417 pg=6.068375 kl=0.004213 reward=1.000000 skipped=145 +[train] epoch=16 step=147 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=146 +[train] epoch=16 step=148 loss=8.117537 pg=8.117537 kl=0.000057 reward=1.000000 skipped=147 +[train] epoch=16 step=149 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=148 +[train] epoch=16 step=150 loss=-7.222335 pg=-7.222339 kl=0.000427 reward=-1.290994 skipped=149 +[train] epoch=16 step=151 loss=6.916299 pg=6.916299 kl=0.000000 reward=1.000000 skipped=150 +[train] epoch=16 step=152 loss=5.446908 pg=5.446908 kl=0.000000 reward=1.000000 skipped=151 +[train] epoch=16 step=153 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=152 +[train] epoch=16 step=154 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=153 +[train] epoch=16 step=155 loss=6.320856 pg=6.320749 kl=0.010641 reward=1.000000 skipped=154 +[train] epoch=16 step=156 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=155 +[train] epoch=16 step=157 loss=5.592641 pg=5.592641 kl=0.000000 reward=1.000000 skipped=156 +[train] epoch=16 step=158 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=157 +[train] epoch=16 step=159 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=158 +[train] epoch=16 step=160 loss=0.000010 pg=0.000000 kl=0.000967 reward=0.000000 skipped=159 +[train] epoch=16 step=161 loss=6.414127 pg=6.414127 kl=0.000000 reward=1.000000 skipped=160 +[train] epoch=16 step=162 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=161 +[train] epoch=16 step=163 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=162 +[train] epoch=16 step=164 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=163 +[train] epoch=16 step=165 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=164 +[train] epoch=16 step=166 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=165 +[train] epoch=16 step=167 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=166 +[train] epoch=16 step=168 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=167 +[train] epoch=16 step=169 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=168 +[train] epoch=16 step=170 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=169 +[train] epoch=16 step=171 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=170 +[train] epoch=16 step=172 loss=5.438523 pg=5.438523 kl=0.000000 reward=1.000000 skipped=171 +[train] epoch=16 step=173 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=172 +[train] epoch=16 step=174 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=173 +[train] epoch=16 step=175 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=174 +[train] epoch=16 step=176 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=175 +[train] epoch=16 step=177 loss=6.916299 pg=6.916299 kl=0.000000 reward=1.000000 skipped=176 +[train] epoch=16 step=178 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=177 +[train] epoch=16 step=179 loss=5.282966 pg=5.282931 kl=0.003500 reward=1.000000 skipped=178 +[train] epoch=16 step=180 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=179 +[train] epoch=16 step=181 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=180 +[train] epoch=16 step=182 loss=6.916299 pg=6.916299 kl=0.000000 reward=1.000000 skipped=181 +[train] epoch=16 step=183 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=182 +[train] epoch=16 step=184 loss=6.414127 pg=6.414127 kl=0.000000 reward=1.000000 skipped=183 +[train] epoch=16 step=185 loss=4.706718 pg=4.706718 kl=0.000000 reward=1.000000 skipped=184 +[train] epoch=16 step=186 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=185 +[train] epoch=16 step=187 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=186 +[train] epoch=16 step=188 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=187 +[train] epoch=16 step=189 loss=5.438523 pg=5.438523 kl=0.000000 reward=1.000000 skipped=188 +[train] epoch=16 step=190 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=189 +[train] epoch=16 step=191 loss=7.161344 pg=7.161344 kl=0.000000 reward=1.000000 skipped=190 +[train] epoch=16 step=192 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=191 +[train] epoch=16 step=193 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=192 +[train] epoch=16 step=194 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=193 +[train] epoch=16 step=195 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=194 +[train] epoch=16 step=196 loss=6.603974 pg=6.603972 kl=0.000205 reward=1.000000 skipped=195 +[train] epoch=16 step=197 loss=-7.222335 pg=-7.222339 kl=0.000427 reward=-1.290994 skipped=196 +[train] epoch=16 step=198 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=197 +[train] epoch=16 step=199 loss=4.211183 pg=4.211183 kl=0.000000 reward=0.774597 skipped=198 +[train] epoch=16 step=200 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=199 +[train] epoch=16 step=201 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=200 +[train] epoch=16 step=202 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=201 +[train] epoch=16 step=203 loss=6.320856 pg=6.320749 kl=0.010641 reward=1.000000 skipped=202 +[train] epoch=16 step=204 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=203 +[train] epoch=16 step=205 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=204 +[train] epoch=16 step=206 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=205 +[train] epoch=16 step=207 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=206 +[train] epoch=16 step=208 loss=6.468365 pg=6.468350 kl=0.001409 reward=1.000000 skipped=207 +[train] epoch=16 step=209 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=208 +[train] epoch=16 step=210 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=209 +[train] epoch=16 step=211 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=210 +[train] epoch=16 step=212 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=211 +[train] epoch=16 step=213 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=212 +[train] epoch=16 step=214 loss=6.603974 pg=6.603972 kl=0.000205 reward=1.000000 skipped=213 +[train] epoch=16 step=215 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=214 +[train] epoch=16 step=216 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=215 +[train] epoch=16 step=217 loss=6.068417 pg=6.068375 kl=0.004213 reward=1.000000 skipped=216 +[train] epoch=16 step=218 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=217 +[train] epoch=16 step=219 loss=5.581098 pg=5.581098 kl=0.000000 reward=1.000000 skipped=218 +[train] epoch=16 step=220 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=219 +[train] epoch=16 step=221 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=220 +[train] epoch=16 step=222 loss=3.049087 pg=3.049040 kl=0.004637 reward=0.577350 skipped=221 +[train] epoch=16 step=223 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=222 +[train] epoch=16 step=224 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=223 +[train] epoch=16 step=225 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=224 +[train] epoch=16 step=226 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=225 +[train] epoch=16 step=227 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=226 +[train] epoch=16 step=228 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=227 +[train] epoch=16 step=229 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=228 +[train] epoch=16 step=230 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=229 +[train] epoch=16 step=231 loss=6.120806 pg=6.120743 kl=0.006336 reward=1.000000 skipped=230 +[train] epoch=16 step=232 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=231 +[train] epoch=16 step=233 loss=6.603974 pg=6.603972 kl=0.000205 reward=1.000000 skipped=232 +[train] epoch=16 step=234 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=233 +[train] epoch=16 step=235 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=234 +[train] epoch=16 step=236 loss=6.746428 pg=6.746428 kl=0.000000 reward=1.000000 skipped=235 +[train] epoch=16 step=237 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=236 +[train] epoch=16 step=238 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=237 +[train] epoch=16 step=239 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=238 +[train] epoch=16 step=240 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=239 +[train] epoch=16 step=241 loss=5.446908 pg=5.446908 kl=0.000000 reward=1.000000 skipped=240 +[train] epoch=16 step=242 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=241 +[train] epoch=16 step=243 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=242 +[train] epoch=16 step=244 loss=8.125317 pg=8.125317 kl=0.000000 reward=1.000000 skipped=243 +[train] epoch=16 step=245 loss=7.926152 pg=7.926152 kl=0.000000 reward=1.000000 skipped=244 +[train] epoch=16 step=246 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=245 +[train] epoch=16 step=247 loss=7.161344 pg=7.161344 kl=0.000000 reward=1.000000 skipped=246 +[train] epoch=16 step=248 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=247 +[train] epoch=16 step=249 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=248 +[train] epoch=16 step=250 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=249 +[train] epoch=16 step=251 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=250 +[train] epoch=16 step=252 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=251 +[train] epoch=16 step=253 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=252 +[train] epoch=16 step=254 loss=5.592641 pg=5.592641 kl=0.000000 reward=1.000000 skipped=253 +[train] epoch=16 step=255 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=254 +[train] epoch=16 step=256 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=255 +[train] epoch=16 step=257 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=256 +[train] epoch=16 step=258 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=257 +[train] epoch=16 step=259 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=258 +[train] epoch=16 step=260 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=259 +[train] epoch=16 step=261 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=260 +[train] epoch=16 step=262 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=261 +[train] epoch=16 step=263 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=262 +[train] epoch=16 step=264 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=263 +[train] epoch=16 step=265 loss=6.320856 pg=6.320749 kl=0.010641 reward=1.000000 skipped=264 +[train] epoch=16 step=266 loss=6.603974 pg=6.603972 kl=0.000205 reward=1.000000 skipped=265 +[train] epoch=16 step=267 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=266 +[train] epoch=16 step=268 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=267 +[train] epoch=16 step=269 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=268 +[train] epoch=16 step=270 loss=4.706718 pg=4.706718 kl=0.000000 reward=1.000000 skipped=269 +[train] epoch=16 step=271 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=270 +[train] epoch=16 step=272 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=271 +[train] epoch=16 step=273 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=272 +[train] epoch=16 step=274 loss=6.603974 pg=6.603972 kl=0.000205 reward=1.000000 skipped=273 +[train] epoch=16 step=275 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=274 +[train] epoch=16 step=276 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=275 +[train] epoch=16 step=277 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=276 +[train] epoch=16 step=278 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=277 +[train] epoch=16 step=279 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=278 +[train] epoch=16 step=280 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=279 +[train] epoch=16 step=281 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=280 +[train] epoch=16 step=282 loss=5.875602 pg=5.875589 kl=0.001239 reward=1.000000 skipped=281 +[train] epoch=16 step=283 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=282 +[train] epoch=16 step=284 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=283 +[train] epoch=16 step=285 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=284 +[train] epoch=16 step=286 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=285 +[train] epoch=16 step=287 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=286 +[train] epoch=16 step=288 loss=6.282383 pg=6.282336 kl=0.004788 reward=1.000000 skipped=287 +[train] epoch=16 step=289 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=288 +[train] epoch=16 step=290 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=289 +[train] epoch=16 step=291 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=290 +[train] epoch=16 step=292 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=291 +[train] epoch=16 step=293 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=292 +[train] epoch=16 step=294 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=293 +[train] epoch=16 step=295 loss=-8.763932 pg=-8.763932 kl=0.000000 reward=-1.290994 skipped=294 +[train] epoch=16 step=296 loss=6.120806 pg=6.120743 kl=0.006336 reward=1.000000 skipped=295 +[train] epoch=16 step=297 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=296 +[train] epoch=16 step=298 loss=0.000013 pg=0.000000 kl=0.001251 reward=0.000000 skipped=297 +[train] epoch=16 step=299 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=298 +[train] epoch=16 step=300 loss=0.000010 pg=0.000000 kl=0.000967 reward=0.000000 skipped=299 +[train] epoch=16 step=301 loss=6.862856 pg=6.862832 kl=0.002382 reward=1.000000 skipped=300 +[train] epoch=16 step=302 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=301 +[train] epoch=16 step=303 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=302 +[train] epoch=16 step=304 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=303 +[train] epoch=16 step=305 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=304 +[train] epoch=16 step=306 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=305 +[train] epoch=16 step=307 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=306 +[train] epoch=16 step=308 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=307 +[train] epoch=16 step=309 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=308 +[train] epoch=16 step=310 loss=4.706718 pg=4.706718 kl=0.000000 reward=1.000000 skipped=309 +[train] epoch=16 step=311 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=310 +[train] epoch=16 step=312 loss=4.211183 pg=4.211183 kl=0.000000 reward=0.774597 skipped=311 +[train] epoch=16 step=313 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=312 +[train] epoch=16 step=314 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=313 +[train] epoch=16 step=315 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=314 +[train] epoch=16 step=316 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=315 +[train] epoch=16 step=317 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=316 +[train] epoch=16 step=318 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=317 +[train] epoch=16 step=319 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=318 +[train] epoch=16 step=320 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=319 +[train] epoch=16 step=321 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=320 +[train] epoch=16 step=322 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=321 +[train] epoch=16 step=323 loss=7.256117 pg=7.256107 kl=0.000982 reward=1.000000 skipped=322 +[train] epoch=16 step=324 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=323 +[train] epoch=16 step=325 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=324 +[train] epoch=16 step=326 loss=5.592641 pg=5.592641 kl=0.000000 reward=1.000000 skipped=325 +[train] epoch=16 step=327 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=326 +[train] epoch=16 step=328 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=327 +[train] epoch=16 step=329 loss=5.369410 pg=5.369405 kl=0.000436 reward=1.000000 skipped=328 +[train] epoch=16 step=330 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=329 +[train] epoch=16 step=331 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=330 +[train] epoch=16 step=332 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=331 +[train] epoch=16 step=333 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=332 +[train] epoch=16 step=334 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=333 +[train] epoch=16 step=335 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=334 +[train] epoch=16 step=336 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=335 +[train] epoch=16 step=337 loss=5.369410 pg=5.369405 kl=0.000436 reward=1.000000 skipped=336 +[train] epoch=16 step=338 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=337 +[train] epoch=16 step=339 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=338 +[train] epoch=16 step=340 loss=-2.338539 pg=-2.338539 kl=0.000000 reward=-0.377964 skipped=339 +[train] epoch=16 step=341 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=340 +[train] epoch=16 step=342 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=341 +[train] epoch=16 step=343 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=342 +[train] epoch=16 step=344 loss=6.320856 pg=6.320749 kl=0.010641 reward=1.000000 skipped=343 +[train] epoch=16 step=345 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=344 +[train] epoch=16 step=346 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=345 +[train] epoch=16 step=347 loss=6.593108 pg=6.593024 kl=0.008324 reward=1.000000 skipped=346 +[train] epoch=16 step=348 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=347 +[train] epoch=16 step=349 loss=5.696458 pg=5.696399 kl=0.005970 reward=1.000000 skipped=348 +[train] epoch=16 step=350 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=349 +[train] epoch=16 step=351 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=350 +[train] epoch=16 step=352 loss=7.021185 pg=7.021152 kl=0.003361 reward=1.000000 skipped=351 +[train] epoch=16 step=353 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=352 +[train] epoch=16 step=354 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=353 +[train] epoch=16 step=355 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=354 +[train] epoch=16 step=356 loss=5.951099 pg=5.951099 kl=0.000000 reward=1.000000 skipped=355 +[train] epoch=16 step=357 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=356 +[train] epoch=16 step=358 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=357 +[train] epoch=16 step=359 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=358 +[train] epoch=16 step=360 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=359 +[train] epoch=16 step=361 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=360 +[train] epoch=16 step=362 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=361 +[train] epoch=16 step=363 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=362 +[train] epoch=16 step=364 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=363 +[train] epoch=16 step=365 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=364 +[train] epoch=16 step=366 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=365 +[train] epoch=16 step=367 loss=4.211183 pg=4.211183 kl=0.000000 reward=0.774597 skipped=366 +[train] epoch=16 step=368 loss=5.696458 pg=5.696399 kl=0.005970 reward=1.000000 skipped=367 +[train] epoch=16 step=369 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=368 +[train] epoch=16 step=370 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=369 +[train] epoch=16 step=371 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=370 +[train] epoch=16 step=372 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=371 +[train] epoch=16 step=373 loss=7.845365 pg=7.845365 kl=0.000000 reward=1.000000 skipped=372 +[train] epoch=16 step=374 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=373 +[train] epoch=16 step=375 loss=8.245316 pg=8.245316 kl=0.000000 reward=1.000000 skipped=374 +[train] epoch=16 step=376 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=375 +[train] epoch=16 step=377 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=376 +[train] epoch=16 step=378 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=377 +[train] epoch=16 step=379 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=378 +[train] epoch=16 step=380 loss=5.951099 pg=5.951099 kl=0.000000 reward=1.000000 skipped=379 +[train] epoch=16 step=381 loss=6.194232 pg=6.194232 kl=0.000000 reward=1.000000 skipped=380 +[train] epoch=16 step=382 loss=5.951652 pg=5.951652 kl=0.000000 reward=1.000000 skipped=381 +[train] epoch=16 step=383 loss=6.194232 pg=6.194232 kl=0.000000 reward=1.000000 skipped=382 +[train] epoch=16 step=384 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=383 +[train] epoch=16 step=385 loss=4.836777 pg=4.836777 kl=0.000000 reward=1.000000 skipped=384 +[train] epoch=16 step=386 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=385 +[train] epoch=16 step=387 loss=7.204127 pg=7.204088 kl=0.003951 reward=1.000000 skipped=386 +[train] epoch=16 step=388 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=387 +[train] epoch=16 step=389 loss=5.734613 pg=5.734613 kl=0.000000 reward=1.000000 skipped=388 +[train] epoch=16 step=390 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=389 +[train] epoch=16 step=391 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=390 +[train] epoch=16 step=392 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=391 +[train] epoch=16 step=393 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=392 +[train] epoch=16 step=394 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=393 +[train] epoch=16 step=395 loss=5.446908 pg=5.446908 kl=0.000000 reward=1.000000 skipped=394 +[train] epoch=16 step=396 loss=6.173695 pg=6.173695 kl=0.000000 reward=1.000000 skipped=395 +[train] epoch=16 step=397 loss=-9.766863 pg=-9.766896 kl=0.003295 reward=-1.732051 skipped=396 +[train] epoch=16 step=398 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=397 +[train] epoch=16 step=399 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=398 +[train] epoch=16 step=400 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=399 +[train] epoch=16 step=401 loss=5.734613 pg=5.734613 kl=0.000000 reward=1.000000 skipped=400 +[train] epoch=16 step=402 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=401 +[train] epoch=16 step=403 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=402 +[train] epoch=16 step=404 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=403 +[train] epoch=16 step=405 loss=6.120806 pg=6.120743 kl=0.006336 reward=1.000000 skipped=404 +[train] epoch=16 step=406 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=405 +[train] epoch=16 step=407 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=406 +[train] epoch=16 step=408 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=407 +[train] epoch=16 step=409 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=408 +[train] epoch=16 step=410 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=409 +[train] epoch=16 step=411 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=410 +[train] epoch=16 step=412 loss=6.578031 pg=6.578004 kl=0.002699 reward=1.000000 skipped=411 +[train] epoch=16 step=413 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=412 +[train] epoch=16 step=414 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=413 +[train] epoch=16 step=415 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=414 +[train] epoch=16 step=416 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=415 +[train] epoch=16 step=417 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=416 +[train] epoch=16 step=418 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=417 +[train] epoch=16 step=419 loss=6.916299 pg=6.916299 kl=0.000000 reward=1.000000 skipped=418 +[train] epoch=16 step=420 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=419 +[train] epoch=16 step=421 loss=0.000010 pg=0.000000 kl=0.000967 reward=0.000000 skipped=420 +[train] epoch=16 step=422 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=421 +[train] epoch=16 step=423 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=422 +[train] epoch=16 step=424 loss=6.746428 pg=6.746428 kl=0.000000 reward=1.000000 skipped=423 +[train] epoch=16 step=425 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=424 +[train] epoch=16 step=426 loss=6.916299 pg=6.916299 kl=0.000000 reward=1.000000 skipped=425 +[train] epoch=16 step=427 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=426 +[train] epoch=16 step=428 loss=5.438523 pg=5.438523 kl=0.000000 reward=1.000000 skipped=427 +[train] epoch=16 step=429 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=428 +[train] epoch=16 step=430 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=429 +[train] epoch=16 step=431 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=430 +[train] epoch=16 step=432 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=431 +[train] epoch=16 step=433 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=432 +[train] epoch=16 step=434 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=433 +[train] epoch=16 step=435 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=434 +[train] epoch=16 step=436 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=435 +[train] epoch=16 step=437 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=436 +[train] epoch=16 step=438 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=437 +[train] epoch=16 step=439 loss=7.161344 pg=7.161344 kl=0.000000 reward=1.000000 skipped=438 +[train] epoch=16 step=440 loss=7.312311 pg=7.312311 kl=0.000000 reward=1.000000 skipped=439 +[train] epoch=16 step=441 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=440 +[train] epoch=16 step=442 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=441 +[train] epoch=16 step=443 loss=7.204127 pg=7.204088 kl=0.003951 reward=1.000000 skipped=442 +[train] epoch=16 step=444 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=443 +[train] epoch=16 step=445 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=444 +[train] epoch=16 step=446 loss=5.696458 pg=5.696399 kl=0.005970 reward=1.000000 skipped=445 +[train] epoch=16 step=447 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=446 +[train] epoch=16 step=448 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=447 +[train] epoch=16 step=449 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=448 +[train] epoch=16 step=450 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=449 +[train] epoch=16 step=451 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=450 +[train] epoch=16 step=452 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=451 +[train] epoch=16 step=453 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=452 +[train] epoch=16 step=454 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=453 +[train] epoch=16 step=455 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=454 +[train] epoch=16 step=456 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=455 +[train] epoch=16 step=457 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=456 +[train] epoch=16 step=458 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=457 +[train] epoch=16 step=459 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=458 +[train] epoch=16 step=460 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=459 +[train] epoch=16 step=461 loss=6.916299 pg=6.916299 kl=0.000000 reward=1.000000 skipped=460 +[train] epoch=16 step=462 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=461 +[train] epoch=16 step=463 loss=7.312311 pg=7.312311 kl=0.000000 reward=1.000000 skipped=462 +[train] epoch=16 step=464 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=463 +[train] epoch=16 step=465 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=464 +[train] epoch=16 step=466 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=465 +[train] epoch=16 step=467 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=466 +[train] epoch=16 step=468 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=467 +[train] epoch=16 step=469 loss=5.792642 pg=5.792642 kl=0.000000 reward=1.000000 skipped=468 +[train] epoch=16 step=470 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=469 +[train] epoch=16 step=471 loss=8.245316 pg=8.245316 kl=0.000000 reward=1.000000 skipped=470 +[train] epoch=16 step=472 loss=6.746428 pg=6.746428 kl=0.000000 reward=1.000000 skipped=471 +[train] epoch=16 step=473 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=472 +[train] epoch=16 step=474 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=473 +[train] epoch=16 step=475 loss=0.000013 pg=0.000000 kl=0.001251 reward=0.000000 skipped=474 +[train] epoch=16 step=476 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=475 +[train] epoch=16 step=477 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=476 +[train] epoch=16 step=478 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=477 +[train] epoch=16 step=479 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=478 +[train] epoch=16 step=480 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=479 +[train] epoch=16 step=481 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=480 +[train] epoch=16 step=482 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=481 +[train] epoch=16 step=483 loss=5.568174 pg=5.568095 kl=0.007986 reward=1.000000 skipped=482 +[train] epoch=16 step=484 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=483 +[train] epoch=16 step=485 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=484 +[train] epoch=16 step=486 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=485 +[train] epoch=16 step=487 loss=5.581098 pg=5.581098 kl=0.000000 reward=1.000000 skipped=486 +[train] epoch=16 step=488 loss=5.581098 pg=5.581098 kl=0.000000 reward=1.000000 skipped=487 +[train] epoch=16 step=489 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=488 +[train] epoch=16 step=490 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=489 +[train] epoch=16 step=491 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=490 +[train] epoch=16 step=492 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=491 +[train] epoch=16 step=493 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=492 +[train] epoch=16 step=494 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=493 +[train] epoch=16 step=495 loss=7.204127 pg=7.204088 kl=0.003951 reward=1.000000 skipped=494 +[train] epoch=16 step=496 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=495 +[train] epoch=16 step=497 loss=5.665429 pg=5.665429 kl=0.000000 reward=0.774597 skipped=496 +[train] epoch=16 step=498 loss=5.951099 pg=5.951099 kl=0.000000 reward=1.000000 skipped=497 +[train] epoch=16 step=499 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=498 +[train] epoch=16 step=500 loss=-7.747314 pg=-7.747314 kl=0.000000 reward=-1.290994 skipped=499 +[train] epoch=16 step=501 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=500 +[train] epoch=16 step=502 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=501 +[train] epoch=16 step=503 loss=5.875602 pg=5.875589 kl=0.001239 reward=1.000000 skipped=502 +[train] epoch=16 step=504 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=503 +[train] epoch=16 step=505 loss=-7.747314 pg=-7.747314 kl=0.000000 reward=-1.290994 skipped=504 +[train] epoch=16 step=506 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=505 +[train] epoch=16 step=507 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=506 +[train] epoch=16 step=508 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=507 +[train] epoch=16 step=509 loss=5.282966 pg=5.282931 kl=0.003500 reward=1.000000 skipped=508 +[train] epoch=16 step=510 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=509 +[train] epoch=16 step=511 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=510 +[train] epoch=16 step=512 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=511 +[train] epoch=16 checkpoint_saved=/data4/runqing/dLLM-RL/rl_llava_next_video_reversal/ckpt/optimized diff --git a/wandb/run-20260601_153643-9x12ylug/files/requirements.txt b/wandb/run-20260601_153643-9x12ylug/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..75f723721d959cc9661b3ed252c3052267b34822 --- /dev/null +++ b/wandb/run-20260601_153643-9x12ylug/files/requirements.txt @@ -0,0 +1,167 @@ +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +tokenizers==0.15.2 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +markdown-it-py==4.2.0 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +accelerate==1.13.0 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +six==1.17.0 +Pygments==2.19.2 +open_clip_torch==3.3.0 +transformers==4.40.0.dev0 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +mdurl==0.1.2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +rich==15.0.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +setuptools==80.10.2 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +peft==0.4.0 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +aiosignal==1.4.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +shtab==1.8.0 +opencv-python-headless==4.9.0.80 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +docstring_parser==0.18.0 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +tyro==0.8.3 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +ftfy==6.3.1 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +timm==1.0.27 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260601_153643-9x12ylug/files/wandb-metadata.json b/wandb/run-20260601_153643-9x12ylug/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..83519c2bb59bc667846e8b164c09f8af4ee108e3 --- /dev/null +++ b/wandb/run-20260601_153643-9x12ylug/files/wandb-metadata.json @@ -0,0 +1,94 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-01T07:36:43.643861Z", + "args": [ + "config=configs/rl_llava_next_video_reversal.yaml", + "wandb.run_id=9x12ylug", + "experiment.current_epoch=16" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_llava_next_video.py", + "codePath": "train/rl_llava_next_video.py", + "codePathLocal": "train/rl_llava_next_video.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/llava-next-video/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "626064564224" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "0m4rjmb3ixcch54jq46xifhzvmedpduo" +} \ No newline at end of file diff --git a/wandb/run-20260601_153643-9x12ylug/files/wandb-summary.json b/wandb/run-20260601_153643-9x12ylug/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d1b80f0f77c3b9401fb1d0d7200edfb404927665 --- /dev/null +++ b/wandb/run-20260601_153643-9x12ylug/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/kl_loss":0,"train/step":512,"train/advantage":1,"train/skipped_steps":511,"train/pg_loss":8.03286361694336,"train/loss":8.03286361694336,"_timestamp":1.7803000488078744e+09,"_runtime":9546,"_step":8191,"_wandb":{"runtime":9546}} \ No newline at end of file diff --git a/wandb/run-20260601_153643-9x12ylug/logs/debug-core.log b/wandb/run-20260601_153643-9x12ylug/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..cd8eca338b3b0067cb923e70458ecd32705de4d4 --- /dev/null +++ b/wandb/run-20260601_153643-9x12ylug/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-06-01T15:36:43.7336128+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpag_bklpk/port-2327850.txt","pid":2327850,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-01T15:36:43.734194575+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2327850-2328039-3640251884/socket","Net":"unix"}} +{"time":"2026-06-01T15:36:43.734313049+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2327850} +{"time":"2026-06-01T15:36:43.904024086+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-01T15:36:43.911126262+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T15:36:44.399006426+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T15:36:49.753978817+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"984pstoygf6n"} +{"time":"2026-06-01T15:48:18.669474283+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"984pstoygf6n"} +{"time":"2026-06-01T15:48:18.889568034+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"984pstoygf6n"} +{"time":"2026-06-01T15:48:18.892304405+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T15:48:18.893186443+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T15:48:18.906820251+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-01T15:48:18.90685825+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-01T15:48:18.906870148+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-01T15:48:18.906903297+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-01T15:48:18.906908465+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-01T15:48:18.906915927+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-01T15:48:18.906979093+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2327850-2328039-3640251884/socket","Net":"unix"}} +{"time":"2026-06-01T15:48:18.906999683+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260601_153643-9x12ylug/logs/debug-internal.log b/wandb/run-20260601_153643-9x12ylug/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5461d43bba05432126a38b60939aff555954c128 --- /dev/null +++ b/wandb/run-20260601_153643-9x12ylug/logs/debug-internal.log @@ -0,0 +1,30 @@ +{"time":"2026-06-01T15:36:43.911255797+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-01T15:36:44.398678602+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-01T15:36:44.398789293+08:00","level":"INFO","msg":"stream: created new stream","id":"9x12ylug"} +{"time":"2026-06-01T15:36:44.39885426+08:00","level":"INFO","msg":"handler: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T15:36:44.398995264+08:00","level":"INFO","msg":"stream: started","id":"9x12ylug"} +{"time":"2026-06-01T15:36:44.398998296+08:00","level":"INFO","msg":"writer: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T15:36:44.399007518+08:00","level":"INFO","msg":"sender: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T15:36:44.401812609+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-01T15:36:49.678926516+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T073649Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2605094d1391294d93cb85d6e5949a2b730759aa0afc2082bd52d1ee5729b6ea"} +{"time":"2026-06-01T15:36:49.679058584+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json18B4E4E172560CB117e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_153643-9x12ylug/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T073649Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2605094d1391294d93cb85d6e5949a2b730759aa0afc2082bd52d1ee5729b6ea, Size: 2704}"} +{"time":"2026-06-01T15:36:49.682914017+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T073649Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=09f79e63851c00aab86bba818d58c9ed532e3edea8b1ba398ee96bf516642605"} +{"time":"2026-06-01T15:36:49.682954291+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txtnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt18B4E4E172E6F9CA4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_153643-9x12ylug/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T073649Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=09f79e63851c00aab86bba818d58c9ed532e3edea8b1ba398ee96bf516642605, Size: 3167}"} +{"time":"2026-06-01T15:36:49.683272404+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T073649Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ee5d0f8754df5fd9bda83862d255fd560055511b4c8df400d53d007305596d7a"} +{"time":"2026-06-01T15:36:49.683391622+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.pynetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py18B4E4E1723BB03617e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_153643-9x12ylug/files/code/train/rl_llava_next_video.py, Name: code/train/rl_llava_next_video.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T073649Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ee5d0f8754df5fd9bda83862d255fd560055511b4c8df400d53d007305596d7a, Size: 15437}"} +{"time":"2026-06-01T15:36:49.686883369+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T073649Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=73853114e9f56fc7bde103861e66385f94d2077aaaa4dffbb7ef5b324a08dd1f"} +{"time":"2026-06-01T15:36:49.68698821+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18B4E4E17261CFCC17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_153643-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T073649Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=73853114e9f56fc7bde103861e66385f94d2077aaaa4dffbb7ef5b324a08dd1f, Size: 14311}"} +{"time":"2026-06-01T15:36:49.688548929+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T073649Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=421aaed49655c0dd44a84e08940c22069fc38fee718ae734fa52d84380a997c9"} +{"time":"2026-06-01T15:36:49.688649938+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch18B4E4E1743BD59E29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_153643-9x12ylug/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T073649Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=421aaed49655c0dd44a84e08940c22069fc38fee718ae734fa52d84380a997c9, Size: 14311}"} +{"time":"2026-06-01T15:48:18.826173442+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T074818Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=90efbce48584ac52d0560012a78008d0647b8d0104dfe325c9392832597076df"} +{"time":"2026-06-01T15:48:18.826271845+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yamlnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml18B4E581E6ADE7CF17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_153643-9x12ylug/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T074818Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=90efbce48584ac52d0560012a78008d0647b8d0104dfe325c9392832597076df, Size: 7149}"} +{"time":"2026-06-01T15:48:18.826864561+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T074818Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b056718edf5749f0b01584697336f048ed3c94d36254bb71cf465cd9fa2833b2"} +{"time":"2026-06-01T15:48:18.826945493+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json18B4E581E6C3124F17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_153643-9x12ylug/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T074818Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b056718edf5749f0b01584697336f048ed3c94d36254bb71cf465cd9fa2833b2, Size: 236}"} +{"time":"2026-06-01T15:48:18.842866446+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T074818Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ad6043748b8c75c2caddcb5789a005077a6c81e24e2d83b296dbad7328c8e830"} +{"time":"2026-06-01T15:48:18.842974421+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.lognetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log18B4E581E7238F944b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_153643-9x12ylug/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T074818Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ad6043748b8c75c2caddcb5789a005077a6c81e24e2d83b296dbad7328c8e830, Size: 48856}"} +{"time":"2026-06-01T15:48:18.843384367+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-01T15:48:18.887534608+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-01T15:48:18.892342093+08:00","level":"INFO","msg":"stream: closing","id":"9x12ylug"} +{"time":"2026-06-01T15:48:18.892370589+08:00","level":"INFO","msg":"handler: closed","stream_id":"9x12ylug"} +{"time":"2026-06-01T15:48:18.892458146+08:00","level":"INFO","msg":"sender: closed","stream_id":"9x12ylug"} +{"time":"2026-06-01T15:48:18.892467834+08:00","level":"INFO","msg":"stream: closed","id":"9x12ylug"} diff --git a/wandb/run-20260601_153643-9x12ylug/logs/debug.log b/wandb/run-20260601_153643-9x12ylug/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3e4cb8b8611486043bd5ecb7b6e255d75c752906 --- /dev/null +++ b/wandb/run-20260601_153643-9x12ylug/logs/debug.log @@ -0,0 +1,25 @@ +2026-06-01 15:36:43,646 INFO MainThread:2327850 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-01 15:36:43,646 INFO MainThread:2327850 [wandb_setup.py:_flush():81] Configure stats pid to 2327850 +2026-06-01 15:36:43,646 INFO MainThread:2327850 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-01 15:36:43,646 INFO MainThread:2327850 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260601_153643-9x12ylug/logs/debug.log +2026-06-01 15:36:43,646 INFO MainThread:2327850 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260601_153643-9x12ylug/logs/debug-internal.log +2026-06-01 15:36:43,646 INFO MainThread:2327850 [wandb_init.py:init():844] calling init triggers +2026-06-01 15:36:43,646 INFO MainThread:2327850 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': '9x12ylug', 'system.env_name': '/data8/runqing/miniconda/envs/llava-next-video', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_llava_next_video_reversal', 'experiment.function': 'train', 'experiment.current_epoch': 16, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.reference_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_name': 'llava_llama', 'model.model_base': None, 'model.conv_mode': 'vicuna_v1', 'model.load_overwrite_config.mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'model.load_overwrite_config.mm_spatial_pool_stride': 2, 'model.load_overwrite_config.mm_spatial_pool_mode': 'average', 'model.load_overwrite_config.mm_newline_position': 'no_token', 'model.load_overwrite_config.max_sequence_length': 8192, 'model.load_overwrite_config.tokenizer_model_max_length': 8192, 'model.load_overwrite_config.rope_scaling.factor': 2.0, 'model.load_overwrite_config.rope_scaling.type': 'linear', 'model.load_overwrite_config.video_fps': 1, 'model.load_overwrite_config.frames_upbound': 32, 'model.load_overwrite_config.force_sample': True, 'model.load_overwrite_config.add_time_instruction': True, 'dataset.train_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_train.json', 'dataset.optimization_data_file': 'rl_data.json', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.top_p': 0.95, 'rollout.max_new_tokens': 128, 'reward.max_direction_gap': 0.2, 'reward.direction_consistency_bonus': 0.3, 'training.num_train_epochs': 1, 'training.gradient_accumulation_steps': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.max_grad_norm': 1.0, 'training.kl_beta': 0.01, 'training.logging_steps': 1, 'training.trainable_parts.0': 'mm_projector', 'training.load_overwrite_config.max_sequence_length': 2048, 'training.load_overwrite_config.tokenizer_model_max_length': 2048, 'training.load_overwrite_config.frames_upbound': 8, 'optimizer.learning_rate': 1e-06, 'optimizer.weight_decay': 0.0, 'evaluation.eval_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json', 'evaluation.num_task_per_step': 64, 'evaluation.num_response_per_task': 3, 'evaluation.max_new_tokens': 128, 'evaluation.temperature': 0.1, 'config': 'configs/rl_llava_next_video_reversal.yaml', '_wandb': {'code_path': 'code/train/rl_llava_next_video.py'}} +2026-06-01 15:36:43,646 INFO MainThread:2327850 [wandb_init.py:init():892] starting backend +2026-06-01 15:36:43,903 INFO MainThread:2327850 [wandb_init.py:init():895] sending inform_init request +2026-06-01 15:36:43,909 INFO MainThread:2327850 [wandb_init.py:init():903] backend started and connected +2026-06-01 15:36:43,911 INFO MainThread:2327850 [wandb_init.py:init():973] updated telemetry +2026-06-01 15:36:43,920 INFO MainThread:2327850 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-01 15:36:44,600 INFO MainThread:2327850 [wandb_init.py:init():1037] run resumed +2026-06-01 15:36:44,602 INFO MainThread:2327850 [wandb_init.py:init():1042] starting run threads in backend +2026-06-01 15:36:44,746 INFO MainThread:2327850 [wandb_run.py:_console_start():2524] atexit reg +2026-06-01 15:36:44,746 INFO MainThread:2327850 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-01 15:36:44,746 INFO MainThread:2327850 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-01 15:36:44,746 INFO MainThread:2327850 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-01 15:36:44,749 INFO MainThread:2327850 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-01 15:48:18,668 INFO MainThread:2327850 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_llava_next_video_reversal/9x12ylug +2026-06-01 15:48:18,668 INFO MainThread:2327850 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-01 15:48:18,669 INFO MainThread:2327850 [wandb_run.py:_restore():2472] restore +2026-06-01 15:48:18,669 INFO MainThread:2327850 [wandb_run.py:_restore():2478] restore done +2026-06-01 15:48:18,891 INFO MainThread:2327850 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260601_170919-9x12ylug/files/code/train/rl_llava_next_video.py b/wandb/run-20260601_170919-9x12ylug/files/code/train/rl_llava_next_video.py new file mode 100644 index 0000000000000000000000000000000000000000..8199163c2bde56855ea46bec0831fe434956172b --- /dev/null +++ b/wandb/run-20260601_170919-9x12ylug/files/code/train/rl_llava_next_video.py @@ -0,0 +1,358 @@ +import copy +import os +import sys +from typing import Iterable, List +import warnings +import random + +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import DataLoader, Dataset +import wandb +from accelerate import Accelerator +from omegaconf import OmegaConf + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LLAVA_NEXT_ROOT = os.path.join(REPO_ROOT, "LLaVA-NeXT") +sys.path.insert(0, LLAVA_NEXT_ROOT) +sys.path.insert(1, REPO_ROOT) + +from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX +from llava.conversation import conv_templates +from llava.mm_utils import tokenizer_image_token +from llava.model.builder import load_pretrained_model +from llava.utils import disable_torch_init, process_video_with_decord + +from train.llava_next_video.common import ( + build_video_mcq_prompt, + ensure_dir, + get_config, + load_json, + resolve_media_abs_path, +) +from train.utils import ensure_wandb_run_id, flatten_omega_conf + +warnings.filterwarnings("ignore", message=".*copying from a non-meta parameter.*") +warnings.filterwarnings("ignore", message=".*Flash Attention 2.0 with a model not initialized on GPU.*") + + +class RLDataset(Dataset): + def __init__(self, samples): + self.samples = samples + + def __len__(self): + return len(self.samples) + + def __getitem__(self, idx): + return self.samples[idx] + + +def build_prompt_text(sample: dict) -> str: + return build_video_mcq_prompt(sample) + + +def tokenize_example(tokenizer, conv_mode: str, prompt_text: str, response_text: str): + prompt_conv = copy.deepcopy(conv_templates[conv_mode]) + prompt_conv.append_message(prompt_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + prompt_conv.append_message(prompt_conv.roles[1], None) + prompt_ids = tokenizer_image_token(prompt_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + full_conv = copy.deepcopy(conv_templates[conv_mode]) + full_conv.append_message(full_conv.roles[0], f"{DEFAULT_IMAGE_TOKEN}\n{prompt_text}") + full_conv.append_message(full_conv.roles[1], response_text) + full_ids = tokenizer_image_token(full_conv.get_prompt(), tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").squeeze(0) + + labels = full_ids.clone() + labels[: prompt_ids.numel()] = -100 + return full_ids, labels + + +def load_video_inputs(sample: dict, image_processor, model_cfg, device): + video_path = resolve_media_abs_path(sample, None, media_type="video") + video, _, _, _ = process_video_with_decord(video_path, model_cfg) + pixel_values = image_processor.preprocess(video, return_tensors="pt")["pixel_values"].to(device=device, dtype=torch.float16) + return [pixel_values], [video[0].size], ["video"] + + +def compute_token_logps(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: + shifted_logits = logits[:, :-1, :] + shifted_labels = labels[:, 1:] + valid_mask = shifted_labels.ne(-100) + gather_labels = shifted_labels.masked_fill(~valid_mask, 0) + token_logps = F.log_softmax(shifted_logits, dim=-1).gather(-1, gather_labels.unsqueeze(-1)).squeeze(-1) + token_logps = token_logps[valid_mask] + return token_logps[torch.isfinite(token_logps)] + + +def safe_mean(values: torch.Tensor, *, device: torch.device, dtype: torch.dtype) -> torch.Tensor: + if values.numel() == 0: + return torch.zeros((), device=device, dtype=dtype) + return values.mean() + + +def snapshot_named_parameters(model, names: Iterable[str]) -> dict[str, torch.Tensor]: + state = {} + target_names = set(names) + for name, param in model.named_parameters(): + if name in target_names: + state[name] = param.detach().cpu().clone() + return state + + +def restore_named_parameters(model, state: dict[str, torch.Tensor]) -> None: + for name, param in model.named_parameters(): + if name in state: + param.data.copy_(state[name].to(device=param.device, dtype=param.dtype)) + + +def gradients_are_finite(params: Iterable[torch.nn.Parameter]) -> bool: + for param in params: + if param.grad is not None and not torch.isfinite(param.grad).all(): + return False + return True + + +def parameters_are_finite(params: Iterable[torch.nn.Parameter]) -> bool: + return all(torch.isfinite(param).all() for param in params) + + +def _merge_overwrite_config(config) -> dict: + overwrite_config = OmegaConf.to_container(config.model.get("load_overwrite_config", {}), resolve=True) or {} + training_overrides = OmegaConf.to_container(config.training.get("load_overwrite_config", {}), resolve=True) or {} + overwrite_config.update(training_overrides) + return overwrite_config + + +def _set_trainable_parameters(model, trainable_parts: Iterable[str]) -> List[torch.nn.Parameter]: + part_to_patterns = { + "lm_head": ["lm_head"], + "embed_tokens": ["embed_tokens"], + "mm_projector": ["mm_projector"], + "vision_resampler": ["vision_resampler"], + "vision_tower": ["vision_tower"], + } + enabled_parts = list(trainable_parts) + enabled_patterns = [] + for part in enabled_parts: + if part == "all": + for param in model.parameters(): + param.requires_grad_(True) + return list(model.parameters()) + enabled_patterns.extend(part_to_patterns.get(part, [part])) + + trainable_params: List[torch.nn.Parameter] = [] + seen_param_ids = set() + for name, param in model.named_parameters(): + should_train = any(pattern in name for pattern in enabled_patterns) + param.requires_grad_(should_train) + if should_train and id(param) not in seen_param_ids: + trainable_params.append(param) + seen_param_ids.add(id(param)) + return trainable_params + + +def _summarize_trainable_parameters(model) -> tuple[int, int, List[str]]: + total_params = 0 + trainable_params = 0 + trainable_names = [] + for name, param in model.named_parameters(): + count = param.numel() + total_params += count + if param.requires_grad: + trainable_params += count + trainable_names.append(name) + return trainable_params, total_params, trainable_names + + +def set_seed(seed: int) -> None: + random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + + +if __name__ == "__main__": + config = get_config() + disable_torch_init() + set_seed(int(config.training.get("seed", 10086))) + if hasattr(torch.backends, "cuda") and torch.cuda.is_available(): + torch.backends.cuda.matmul.allow_tf32 = bool(config.training.get("enable_tf32", True)) + if hasattr(torch.backends, "cudnn"): + torch.backends.cudnn.allow_tf32 = bool(config.training.get("enable_tf32", True)) + + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + ) + + existing_run_id = ensure_wandb_run_id(config) + if accelerator.is_main_process: + wandb.init( + project=config.experiment.project, + id=existing_run_id, + resume=config.wandb.resume, + config={k: v for k, v in flatten_omega_conf(config, resolve=True)}, + name=config.experiment.project, + reinit=True, + ) + + current_epoch = int(config.experiment.current_epoch) + model_path = config.model.pretrained_model if current_epoch == 1 else os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ref_model_path = config.model.reference_model or config.model.pretrained_model + train_samples = load_json(os.path.join(config.experiment.project, "temp_data", config.dataset.optimization_data_file)) + overwrite_config = _merge_overwrite_config(config) + ref_device = torch.device("cuda:1") if torch.cuda.is_available() and torch.cuda.device_count() > 1 else accelerator.device + if accelerator.is_main_process: + print( + f"[train] epoch={current_epoch} samples={len(train_samples)} " + f"grad_accum={int(config.training.gradient_accumulation_steps)}", + flush=True, + ) + + tokenizer, model, image_processor, _ = load_pretrained_model( + model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + _, ref_model, _, _ = load_pretrained_model( + ref_model_path, + config.model.get("model_base", None), + config.model.model_name, + device_map=None, + overwrite_config=overwrite_config if overwrite_config else None, + ) + ref_model.to(device=ref_device, dtype=torch.float16) + ref_model.eval() + for p in ref_model.parameters(): + p.requires_grad_(False) + + trainable_parts = list(config.training.get("trainable_parts", ["lm_head"])) + trainable_params = _set_trainable_parameters(model, trainable_parts) + if not trainable_params: + raise ValueError(f"No trainable parameters matched training.trainable_parts={trainable_parts}") + + trainable_count, total_count, trainable_names = _summarize_trainable_parameters(model) + if accelerator.is_main_process: + print( + f"RL trainable parts: {trainable_parts}; " + f"trainable params={trainable_count:,}/{total_count:,} " + f"({100.0 * trainable_count / max(total_count, 1):.4f}%)" + ) + for name in trainable_names[:20]: + print(f" trainable: {name}") + if len(trainable_names) > 20: + print(f" ... and {len(trainable_names) - 20} more") + + optimizer = AdamW(trainable_params, lr=float(config.optimizer.learning_rate), weight_decay=float(config.optimizer.weight_decay)) + dataloader = DataLoader(RLDataset(train_samples), batch_size=1, shuffle=True) + + model, optimizer, dataloader = accelerator.prepare(model, optimizer, dataloader) + last_good_state = snapshot_named_parameters(accelerator.unwrap_model(model), trainable_names) + model.train() + + global_step = 0 + skipped_steps = 0 + for _ in range(int(config.training.num_train_epochs)): + for batch in dataloader: + sample = {k: v[0] if isinstance(v, list) else v for k, v in batch.items()} + prompt_text = build_prompt_text(sample) + input_ids, labels = tokenize_example(tokenizer, config.model.conv_mode, prompt_text, sample["response"]) + input_ids = input_ids.unsqueeze(0).to(accelerator.device) + labels = labels.unsqueeze(0).to(accelerator.device) + attention_mask = input_ids.ne(tokenizer.pad_token_id if tokenizer.pad_token_id is not None else 0).long() + images, image_sizes, modalities = load_video_inputs(sample, image_processor, model.config, accelerator.device) + ref_input_ids = input_ids.to(ref_device) + ref_attention_mask = attention_mask.to(ref_device) + ref_images = [img.to(ref_device) for img in images] + ref_labels = labels.to(ref_device) + + with accelerator.accumulate(model): + outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + images=images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + with torch.no_grad(): + ref_outputs = ref_model( + input_ids=ref_input_ids, + attention_mask=ref_attention_mask, + images=ref_images, + image_sizes=image_sizes, + modalities=modalities, + use_cache=False, + ) + + policy_logps = compute_token_logps(outputs.logits, labels) + ref_logps = compute_token_logps(ref_outputs.logits, ref_labels).to(policy_logps.device) + shared_tokens = min(policy_logps.numel(), ref_logps.numel()) + if shared_tokens == 0: + skipped_steps += 1 + optimizer.zero_grad() + continue + policy_logps = policy_logps[:shared_tokens] + ref_logps = ref_logps[:shared_tokens] + advantage = torch.tensor(float(sample["reward"]), device=accelerator.device, dtype=policy_logps.dtype) + pg_loss = -(advantage * safe_mean(policy_logps, device=accelerator.device, dtype=policy_logps.dtype)) + kl_loss = torch.clamp( + safe_mean(policy_logps - ref_logps, device=accelerator.device, dtype=policy_logps.dtype), + min=0.0, + ) + loss = pg_loss + float(config.training.kl_beta) * kl_loss + if not torch.isfinite(loss): + skipped_steps += 1 + optimizer.zero_grad() + continue + + accelerator.backward(loss) + if not gradients_are_finite(trainable_params): + skipped_steps += 1 + optimizer.zero_grad() + restore_named_parameters(accelerator.unwrap_model(model), last_good_state) + continue + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, float(config.training.max_grad_norm)) + optimizer.step() + if not parameters_are_finite(trainable_params): + skipped_steps += 1 + restore_named_parameters(accelerator.unwrap_model(model), last_good_state) + optimizer.zero_grad() + continue + last_good_state = snapshot_named_parameters(accelerator.unwrap_model(model), trainable_names) + optimizer.zero_grad() + + global_step += 1 + if accelerator.is_main_process and global_step % int(config.training.logging_steps) == 0: + print( + f"[train] epoch={current_epoch} step={global_step} " + f"loss={float(loss.detach().cpu()):.6f} " + f"pg={float(pg_loss.detach().cpu()):.6f} " + f"kl={float(kl_loss.detach().cpu()):.6f} " + f"reward={float(advantage.detach().cpu()):.6f} " + f"skipped={skipped_steps}", + flush=True, + ) + wandb.log( + { + "train/loss": float(loss.detach().cpu()), + "train/pg_loss": float(pg_loss.detach().cpu()), + "train/kl_loss": float(kl_loss.detach().cpu()), + "train/advantage": float(advantage.detach().cpu()), + "train/skipped_steps": skipped_steps, + "train/step": global_step, + } + ) + + accelerator.wait_for_everyone() + if accelerator.is_main_process: + out_dir = os.path.join(config.experiment.project, "ckpt", config.model.optimized_name) + ensure_dir(out_dir) + accelerator.unwrap_model(model).save_pretrained(out_dir, safe_serialization=True) + tokenizer.save_pretrained(out_dir) + print(f"[train] epoch={current_epoch} checkpoint_saved={os.path.abspath(out_dir)}", flush=True) + wandb.finish() diff --git a/wandb/run-20260601_170919-9x12ylug/files/config.yaml b/wandb/run-20260601_170919-9x12ylug/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d3ed81a624da1a57e71b97bb05de7b1059cfd00 --- /dev/null +++ b/wandb/run-20260601_170919-9x12ylug/files/config.yaml @@ -0,0 +1,223 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_llava_next_video.py + e: + 63ljj4arc5l9774b36d31nxbetbv5xgb: + args: + - config=configs/rl_llava_next_video_reversal.yaml + - wandb.run_id=9x12ylug + - experiment.current_epoch=22 + codePath: train/rl_llava_next_video.py + codePathLocal: train/rl_llava_next_video.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "626407727104" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/llava-next-video/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_llava_next_video.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-06-01T09:09:19.545899Z" + writerId: 63ljj4arc5l9774b36d31nxbetbv5xgb + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + "2": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.40.0.dev0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_llava_next_video_reversal.yaml +dataset.optimization_data_file: + value: rl_data.json +dataset.train_json: + value: /data4/runqing/dLLM-RL/data/video_reversal_rl_train.json +evaluation.eval_json: + value: /data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json +evaluation.max_new_tokens: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.num_task_per_step: + value: 64 +evaluation.temperature: + value: 0.1 +experiment.current_epoch: + value: 22 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.project: + value: rl_llava_next_video_reversal +experiment.save_every: + value: 10 +experiment.total_step: + value: 30 +model.conv_mode: + value: vicuna_v1 +model.load_overwrite_config.add_time_instruction: + value: true +model.load_overwrite_config.force_sample: + value: true +model.load_overwrite_config.frames_upbound: + value: 32 +model.load_overwrite_config.max_sequence_length: + value: 8192 +model.load_overwrite_config.mm_newline_position: + value: no_token +model.load_overwrite_config.mm_spatial_pool_mode: + value: average +model.load_overwrite_config.mm_spatial_pool_stride: + value: 2 +model.load_overwrite_config.mm_vision_tower: + value: /data4/runqing/hf/clip-vit-large-patch14-336 +model.load_overwrite_config.rope_scaling.factor: + value: 2 +model.load_overwrite_config.rope_scaling.type: + value: linear +model.load_overwrite_config.tokenizer_model_max_length: + value: 8192 +model.load_overwrite_config.video_fps: + value: 1 +model.model_base: + value: null +model.model_name: + value: llava_llama +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +model.reference_model: + value: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +optimizer.learning_rate: + value: 1e-06 +optimizer.weight_decay: + value: 0 +reward.direction_consistency_bonus: + value: 0.3 +reward.max_direction_gap: + value: 0.2 +rollout.max_new_tokens: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.temperature: + value: 0.8 +rollout.top_p: + value: 0.95 +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/llava-next-video +training.enable_tf32: + value: true +training.gradient_accumulation_steps: + value: 2 +training.kl_beta: + value: 0.01 +training.load_overwrite_config.frames_upbound: + value: 8 +training.load_overwrite_config.max_sequence_length: + value: 2048 +training.load_overwrite_config.tokenizer_model_max_length: + value: 2048 +training.logging_steps: + value: 1 +training.max_grad_norm: + value: 1 +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.seed: + value: 10086 +training.trainable_parts.0: + value: mm_projector +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: 9x12ylug diff --git a/wandb/run-20260601_170919-9x12ylug/files/diff.patch b/wandb/run-20260601_170919-9x12ylug/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260601_170919-9x12ylug/files/diff.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260601_170919-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260601_170919-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0534e4c76d23528b4a2896b3f14238f3ff1d0a0 --- /dev/null +++ b/wandb/run-20260601_170919-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,355 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260601_170919-9x12ylug/files/output.log b/wandb/run-20260601_170919-9x12ylug/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c91aa99de4a17da8833bbf59cd83628d1aae7d27 --- /dev/null +++ b/wandb/run-20260601_170919-9x12ylug/files/output.log @@ -0,0 +1,529 @@ +[train] epoch=22 samples=1024 grad_accum=2 +Loaded LLaVA model: rl_llava_next_video_reversal/ckpt/optimized +Overwriting config with {'mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'mm_spatial_pool_stride': 2, 'mm_spatial_pool_mode': 'average', 'mm_newline_position': 'no_token', 'max_sequence_length': 2048, 'tokenizer_model_max_length': 2048, 'rope_scaling': {'factor': 2.0, 'type': 'linear'}, 'video_fps': 1, 'frames_upbound': 8, 'force_sample': True, 'add_time_instruction': True} +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +The checkpoint seems to contain `vision_tower` weights: `mm_tunable_parts` contains `mm_vision_tower`. +Model Class: LlavaLlamaForCausalLM +Loaded LLaVA model: /data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized +You are using a model of type llava to instantiate a model of type llava_llama. This is not supported for all configurations of models and can yield errors. +Overwriting config with {'mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'mm_spatial_pool_stride': 2, 'mm_spatial_pool_mode': 'average', 'mm_newline_position': 'no_token', 'max_sequence_length': 2048, 'tokenizer_model_max_length': 2048, 'rope_scaling': {'factor': 2.0, 'type': 'linear'}, 'video_fps': 1, 'frames_upbound': 8, 'force_sample': True, 'add_time_instruction': True} +The checkpoint seems to contain `vision_tower` weights: `mm_tunable_parts` contains `mm_vision_tower`. +Model Class: LlavaLlamaForCausalLM +RL trainable parts: ['mm_projector']; trainable params=20,979,712/7,062,915,072 (0.2970%) + trainable: model.mm_projector.0.weight + trainable: model.mm_projector.0.bias + trainable: model.mm_projector.2.weight + trainable: model.mm_projector.2.bias +[train] epoch=22 step=1 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=0 +[train] epoch=22 step=2 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=1 +[train] epoch=22 step=3 loss=6.282383 pg=6.282336 kl=0.004788 reward=1.000000 skipped=2 +[train] epoch=22 step=4 loss=6.468365 pg=6.468350 kl=0.001409 reward=1.000000 skipped=3 +[train] epoch=22 step=5 loss=7.083083 pg=7.083083 kl=0.000000 reward=1.000000 skipped=4 +[train] epoch=22 step=6 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=5 +[train] epoch=22 step=7 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=6 +[train] epoch=22 step=8 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=7 +[train] epoch=22 step=9 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=8 +[train] epoch=22 step=10 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=9 +[train] epoch=22 step=11 loss=1.996112 pg=1.996065 kl=0.004637 reward=0.377964 skipped=10 +[train] epoch=22 step=12 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=11 +[train] epoch=22 step=13 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=12 +[train] epoch=22 step=14 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=13 +[train] epoch=22 step=15 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=14 +[train] epoch=22 step=16 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=15 +[train] epoch=22 step=17 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=16 +[train] epoch=22 step=18 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=17 +[train] epoch=22 step=19 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=18 +[train] epoch=22 step=20 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=19 +[train] epoch=22 step=21 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=20 +[train] epoch=22 step=22 loss=7.845365 pg=7.845365 kl=0.000000 reward=1.000000 skipped=21 +[train] epoch=22 step=23 loss=7.083083 pg=7.083083 kl=0.000000 reward=1.000000 skipped=22 +[train] epoch=22 step=24 loss=5.951652 pg=5.951652 kl=0.000000 reward=1.000000 skipped=23 +[train] epoch=22 step=25 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=24 +[train] epoch=22 step=26 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=25 +[train] epoch=22 step=27 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=26 +[train] epoch=22 step=28 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=27 +[train] epoch=22 step=29 loss=4.836777 pg=4.836777 kl=0.000000 reward=1.000000 skipped=28 +[train] epoch=22 step=30 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=29 +[train] epoch=22 step=31 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=30 +[train] epoch=22 step=32 loss=5.878745 pg=5.878745 kl=0.000000 reward=1.000000 skipped=31 +[train] epoch=22 step=33 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=32 +[train] epoch=22 step=34 loss=7.771910 pg=7.771876 kl=0.003357 reward=1.000000 skipped=33 +[train] epoch=22 step=35 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=34 +[train] epoch=22 step=36 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=35 +[train] epoch=22 step=37 loss=8.125317 pg=8.125317 kl=0.000000 reward=1.000000 skipped=36 +[train] epoch=22 step=38 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=37 +[train] epoch=22 step=39 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=38 +[train] epoch=22 step=40 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=39 +[train] epoch=22 step=41 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=40 +[train] epoch=22 step=42 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=41 +[train] epoch=22 step=43 loss=1.996112 pg=1.996065 kl=0.004637 reward=0.377964 skipped=42 +[train] epoch=22 step=44 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=43 +[train] epoch=22 step=45 loss=5.151323 pg=5.151323 kl=0.000000 reward=1.000000 skipped=44 +[train] epoch=22 step=46 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=45 +[train] epoch=22 step=47 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=46 +[train] epoch=22 step=48 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=47 +[train] epoch=22 step=49 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=48 +[train] epoch=22 step=50 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=49 +[train] epoch=22 step=51 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=50 +[train] epoch=22 step=52 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=51 +[train] epoch=22 step=53 loss=-2.338539 pg=-2.338539 kl=0.000000 reward=-0.377964 skipped=52 +[train] epoch=22 step=54 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=53 +[train] epoch=22 step=55 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=54 +[train] epoch=22 step=56 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=55 +[train] epoch=22 step=57 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=56 +[train] epoch=22 step=58 loss=4.222761 pg=4.222761 kl=0.000000 reward=0.577350 skipped=57 +[train] epoch=22 step=59 loss=5.369410 pg=5.369405 kl=0.000436 reward=1.000000 skipped=58 +[train] epoch=22 step=60 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=59 +[train] epoch=22 step=61 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=60 +[train] epoch=22 step=62 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=61 +[train] epoch=22 step=63 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=62 +[train] epoch=22 step=64 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=63 +[train] epoch=22 step=65 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=64 +[train] epoch=22 step=66 loss=-8.813005 pg=-8.813011 kl=0.000536 reward=-1.290994 skipped=65 +[train] epoch=22 step=67 loss=5.568174 pg=5.568095 kl=0.007986 reward=1.000000 skipped=66 +[train] epoch=22 step=68 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=67 +[train] epoch=22 step=69 loss=6.862856 pg=6.862832 kl=0.002382 reward=1.000000 skipped=68 +[train] epoch=22 step=70 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=69 +[train] epoch=22 step=71 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=70 +[train] epoch=22 step=72 loss=6.468365 pg=6.468350 kl=0.001409 reward=1.000000 skipped=71 +[train] epoch=22 step=73 loss=8.117537 pg=8.117537 kl=0.000057 reward=1.000000 skipped=72 +[train] epoch=22 step=74 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=73 +[train] epoch=22 step=75 loss=8.125317 pg=8.125317 kl=0.000000 reward=1.000000 skipped=74 +[train] epoch=22 step=76 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=75 +[train] epoch=22 step=77 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=76 +[train] epoch=22 step=78 loss=5.282966 pg=5.282931 kl=0.003500 reward=1.000000 skipped=77 +[train] epoch=22 step=79 loss=5.875602 pg=5.875589 kl=0.001239 reward=1.000000 skipped=78 +[train] epoch=22 step=80 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=79 +[train] epoch=22 step=81 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=80 +[train] epoch=22 step=82 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=81 +[train] epoch=22 step=83 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=82 +[train] epoch=22 step=84 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=83 +[train] epoch=22 step=85 loss=7.771910 pg=7.771876 kl=0.003357 reward=1.000000 skipped=84 +[train] epoch=22 step=86 loss=6.194232 pg=6.194232 kl=0.000000 reward=1.000000 skipped=85 +[train] epoch=22 step=87 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=86 +[train] epoch=22 step=88 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=87 +[train] epoch=22 step=89 loss=4.836777 pg=4.836777 kl=0.000000 reward=1.000000 skipped=88 +[train] epoch=22 step=90 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=89 +[train] epoch=22 step=91 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=90 +[train] epoch=22 step=92 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=91 +[train] epoch=22 step=93 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=92 +[train] epoch=22 step=94 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=93 +[train] epoch=22 step=95 loss=7.021185 pg=7.021152 kl=0.003361 reward=1.000000 skipped=94 +[train] epoch=22 step=96 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=95 +[train] epoch=22 step=97 loss=7.256117 pg=7.256107 kl=0.000982 reward=1.000000 skipped=96 +[train] epoch=22 step=98 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=97 +[train] epoch=22 step=99 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=98 +[train] epoch=22 step=100 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=99 +[train] epoch=22 step=101 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=100 +[train] epoch=22 step=102 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=101 +[train] epoch=22 step=103 loss=6.126596 pg=6.126578 kl=0.001824 reward=1.000000 skipped=102 +[train] epoch=22 step=104 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=103 +[train] epoch=22 step=105 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=104 +[train] epoch=22 step=106 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=105 +[train] epoch=22 step=107 loss=6.295836 pg=6.295836 kl=0.000000 reward=1.000000 skipped=106 +[train] epoch=22 step=108 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=107 +[train] epoch=22 step=109 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=108 +[train] epoch=22 step=110 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=109 +[train] epoch=22 step=111 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=110 +[train] epoch=22 step=112 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=111 +[train] epoch=22 step=113 loss=6.295836 pg=6.295836 kl=0.000000 reward=1.000000 skipped=112 +[train] epoch=22 step=114 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=113 +[train] epoch=22 step=115 loss=6.295836 pg=6.295836 kl=0.000000 reward=1.000000 skipped=114 +[train] epoch=22 step=116 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=115 +[train] epoch=22 step=117 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=116 +[train] epoch=22 step=118 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=117 +[train] epoch=22 step=119 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=118 +[train] epoch=22 step=120 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=119 +[train] epoch=22 step=121 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=120 +[train] epoch=22 step=122 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=121 +[train] epoch=22 step=123 loss=7.845365 pg=7.845365 kl=0.000000 reward=1.000000 skipped=122 +[train] epoch=22 step=124 loss=6.414127 pg=6.414127 kl=0.000000 reward=1.000000 skipped=123 +[train] epoch=22 step=125 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=124 +[train] epoch=22 step=126 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=125 +[train] epoch=22 step=127 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=126 +[train] epoch=22 step=128 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=127 +[train] epoch=22 step=129 loss=6.862856 pg=6.862832 kl=0.002382 reward=1.000000 skipped=128 +[train] epoch=22 step=130 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=129 +[train] epoch=22 step=131 loss=7.021185 pg=7.021152 kl=0.003361 reward=1.000000 skipped=130 +[train] epoch=22 step=132 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=131 +[train] epoch=22 step=133 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=132 +[train] epoch=22 step=134 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=133 +[train] epoch=22 step=135 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=134 +[train] epoch=22 step=136 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=135 +[train] epoch=22 step=137 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=136 +[train] epoch=22 step=138 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=137 +[train] epoch=22 step=139 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=138 +[train] epoch=22 step=140 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=139 +[train] epoch=22 step=141 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=140 +[train] epoch=22 step=142 loss=5.734613 pg=5.734613 kl=0.000000 reward=1.000000 skipped=141 +[train] epoch=22 step=143 loss=5.878745 pg=5.878745 kl=0.000000 reward=1.000000 skipped=142 +[train] epoch=22 step=144 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=143 +[train] epoch=22 step=145 loss=-2.338539 pg=-2.338539 kl=0.000000 reward=-0.377964 skipped=144 +[train] epoch=22 step=146 loss=6.068417 pg=6.068375 kl=0.004213 reward=1.000000 skipped=145 +[train] epoch=22 step=147 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=146 +[train] epoch=22 step=148 loss=8.117537 pg=8.117537 kl=0.000057 reward=1.000000 skipped=147 +[train] epoch=22 step=149 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=148 +[train] epoch=22 step=150 loss=4.211183 pg=4.211183 kl=0.000000 reward=0.774597 skipped=149 +[train] epoch=22 step=151 loss=2.614115 pg=2.614115 kl=0.000000 reward=0.377964 skipped=150 +[train] epoch=22 step=152 loss=5.446908 pg=5.446908 kl=0.000000 reward=1.000000 skipped=151 +[train] epoch=22 step=153 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=152 +[train] epoch=22 step=154 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=153 +[train] epoch=22 step=155 loss=2.389125 pg=2.389019 kl=0.010641 reward=0.377964 skipped=154 +[train] epoch=22 step=156 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=155 +[train] epoch=22 step=157 loss=5.592641 pg=5.592641 kl=0.000000 reward=1.000000 skipped=156 +[train] epoch=22 step=158 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=157 +[train] epoch=22 step=159 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=158 +[train] epoch=22 step=160 loss=0.000010 pg=0.000000 kl=0.000967 reward=0.000000 skipped=159 +[train] epoch=22 step=161 loss=6.414127 pg=6.414127 kl=0.000000 reward=1.000000 skipped=160 +[train] epoch=22 step=162 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=161 +[train] epoch=22 step=163 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=162 +[train] epoch=22 step=164 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=163 +[train] epoch=22 step=165 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=164 +[train] epoch=22 step=166 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=165 +[train] epoch=22 step=167 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=166 +[train] epoch=22 step=168 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=167 +[train] epoch=22 step=169 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=168 +[train] epoch=22 step=170 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=169 +[train] epoch=22 step=171 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=170 +[train] epoch=22 step=172 loss=5.438523 pg=5.438523 kl=0.000000 reward=1.000000 skipped=171 +[train] epoch=22 step=173 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=172 +[train] epoch=22 step=174 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=173 +[train] epoch=22 step=175 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=174 +[train] epoch=22 step=176 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=175 +[train] epoch=22 step=177 loss=2.614115 pg=2.614115 kl=0.000000 reward=0.377964 skipped=176 +[train] epoch=22 step=178 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=177 +[train] epoch=22 step=179 loss=5.282966 pg=5.282931 kl=0.003500 reward=1.000000 skipped=178 +[train] epoch=22 step=180 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=179 +[train] epoch=22 step=181 loss=6.199073 pg=6.199073 kl=0.000000 reward=1.000000 skipped=180 +[train] epoch=22 step=182 loss=2.614115 pg=2.614115 kl=0.000000 reward=0.377964 skipped=181 +[train] epoch=22 step=183 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=182 +[train] epoch=22 step=184 loss=6.414127 pg=6.414127 kl=0.000000 reward=1.000000 skipped=183 +[train] epoch=22 step=185 loss=4.706718 pg=4.706718 kl=0.000000 reward=1.000000 skipped=184 +[train] epoch=22 step=186 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=185 +[train] epoch=22 step=187 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=186 +[train] epoch=22 step=188 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=187 +[train] epoch=22 step=189 loss=5.438523 pg=5.438523 kl=0.000000 reward=1.000000 skipped=188 +[train] epoch=22 step=190 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=189 +[train] epoch=22 step=191 loss=7.161344 pg=7.161344 kl=0.000000 reward=1.000000 skipped=190 +[train] epoch=22 step=192 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=191 +[train] epoch=22 step=193 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=192 +[train] epoch=22 step=194 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=193 +[train] epoch=22 step=195 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=194 +[train] epoch=22 step=196 loss=-17.953377 pg=-17.953447 kl=0.007016 reward=-2.645751 skipped=195 +[train] epoch=22 step=197 loss=4.211183 pg=4.211183 kl=0.000000 reward=0.774597 skipped=196 +[train] epoch=22 step=198 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=197 +[train] epoch=22 step=199 loss=4.211183 pg=4.211183 kl=0.000000 reward=0.774597 skipped=198 +[train] epoch=22 step=200 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=199 +[train] epoch=22 step=201 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=200 +[train] epoch=22 step=202 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=201 +[train] epoch=22 step=203 loss=2.389125 pg=2.389019 kl=0.010641 reward=0.377964 skipped=202 +[train] epoch=22 step=204 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=203 +[train] epoch=22 step=205 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=204 +[train] epoch=22 step=206 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=205 +[train] epoch=22 step=207 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=206 +[train] epoch=22 step=208 loss=6.468365 pg=6.468350 kl=0.001409 reward=1.000000 skipped=207 +[train] epoch=22 step=209 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=208 +[train] epoch=22 step=210 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=209 +[train] epoch=22 step=211 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=210 +[train] epoch=22 step=212 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=211 +[train] epoch=22 step=213 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=212 +[train] epoch=22 step=214 loss=2.496069 pg=2.496067 kl=0.000205 reward=0.377964 skipped=213 +[train] epoch=22 step=215 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=214 +[train] epoch=22 step=216 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=215 +[train] epoch=22 step=217 loss=6.068417 pg=6.068375 kl=0.004213 reward=1.000000 skipped=216 +[train] epoch=22 step=218 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=217 +[train] epoch=22 step=219 loss=5.581098 pg=5.581098 kl=0.000000 reward=1.000000 skipped=218 +[train] epoch=22 step=220 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=219 +[train] epoch=22 step=221 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=220 +[train] epoch=22 step=222 loss=1.996112 pg=1.996065 kl=0.004637 reward=0.377964 skipped=221 +[train] epoch=22 step=223 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=222 +[train] epoch=22 step=224 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=223 +[train] epoch=22 step=225 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=224 +[train] epoch=22 step=226 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=225 +[train] epoch=22 step=227 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=226 +[train] epoch=22 step=228 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=227 +[train] epoch=22 step=229 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=228 +[train] epoch=22 step=230 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=229 +[train] epoch=22 step=231 loss=6.120806 pg=6.120743 kl=0.006336 reward=1.000000 skipped=230 +[train] epoch=22 step=232 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=231 +[train] epoch=22 step=233 loss=2.496069 pg=2.496067 kl=0.000205 reward=0.377964 skipped=232 +[train] epoch=22 step=234 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=233 +[train] epoch=22 step=235 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=234 +[train] epoch=22 step=236 loss=6.746428 pg=6.746428 kl=0.000000 reward=1.000000 skipped=235 +[train] epoch=22 step=237 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=236 +[train] epoch=22 step=238 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=237 +[train] epoch=22 step=239 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=238 +[train] epoch=22 step=240 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=239 +[train] epoch=22 step=241 loss=5.446908 pg=5.446908 kl=0.000000 reward=1.000000 skipped=240 +[train] epoch=22 step=242 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=241 +[train] epoch=22 step=243 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=242 +[train] epoch=22 step=244 loss=8.125317 pg=8.125317 kl=0.000000 reward=1.000000 skipped=243 +[train] epoch=22 step=245 loss=7.926152 pg=7.926152 kl=0.000000 reward=1.000000 skipped=244 +[train] epoch=22 step=246 loss=6.997608 pg=6.997608 kl=0.000000 reward=1.000000 skipped=245 +[train] epoch=22 step=247 loss=7.161344 pg=7.161344 kl=0.000000 reward=1.000000 skipped=246 +[train] epoch=22 step=248 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=247 +[train] epoch=22 step=249 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=248 +[train] epoch=22 step=250 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=249 +[train] epoch=22 step=251 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=250 +[train] epoch=22 step=252 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=251 +[train] epoch=22 step=253 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=252 +[train] epoch=22 step=254 loss=5.592641 pg=5.592641 kl=0.000000 reward=1.000000 skipped=253 +[train] epoch=22 step=255 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=254 +[train] epoch=22 step=256 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=255 +[train] epoch=22 step=257 loss=6.443498 pg=6.443498 kl=0.000000 reward=1.000000 skipped=256 +[train] epoch=22 step=258 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=257 +[train] epoch=22 step=259 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=258 +[train] epoch=22 step=260 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=259 +[train] epoch=22 step=261 loss=5.899285 pg=5.899285 kl=0.000000 reward=1.000000 skipped=260 +[train] epoch=22 step=262 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=261 +[train] epoch=22 step=263 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=262 +[train] epoch=22 step=264 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=263 +[train] epoch=22 step=265 loss=2.389125 pg=2.389019 kl=0.010641 reward=0.377964 skipped=264 +[train] epoch=22 step=266 loss=2.496069 pg=2.496067 kl=0.000205 reward=0.377964 skipped=265 +[train] epoch=22 step=267 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=266 +[train] epoch=22 step=268 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=267 +[train] epoch=22 step=269 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=268 +[train] epoch=22 step=270 loss=4.706718 pg=4.706718 kl=0.000000 reward=1.000000 skipped=269 +[train] epoch=22 step=271 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=270 +[train] epoch=22 step=272 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=271 +[train] epoch=22 step=273 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=272 +[train] epoch=22 step=274 loss=2.496069 pg=2.496067 kl=0.000205 reward=0.377964 skipped=273 +[train] epoch=22 step=275 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=274 +[train] epoch=22 step=276 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=275 +[train] epoch=22 step=277 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=276 +[train] epoch=22 step=278 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=277 +[train] epoch=22 step=279 loss=6.523762 pg=6.523743 kl=0.001915 reward=1.000000 skipped=278 +[train] epoch=22 step=280 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=279 +[train] epoch=22 step=281 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=280 +[train] epoch=22 step=282 loss=5.875602 pg=5.875589 kl=0.001239 reward=1.000000 skipped=281 +[train] epoch=22 step=283 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=282 +[train] epoch=22 step=284 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=283 +[train] epoch=22 step=285 loss=7.145880 pg=7.145880 kl=0.000000 reward=1.000000 skipped=284 +[train] epoch=22 step=286 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=285 +[train] epoch=22 step=287 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=286 +[train] epoch=22 step=288 loss=6.282383 pg=6.282336 kl=0.004788 reward=1.000000 skipped=287 +[train] epoch=22 step=289 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=288 +[train] epoch=22 step=290 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=289 +[train] epoch=22 step=291 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=290 +[train] epoch=22 step=292 loss=6.170984 pg=6.170984 kl=0.000000 reward=1.000000 skipped=291 +[train] epoch=22 step=293 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=292 +[train] epoch=22 step=294 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=293 +[train] epoch=22 step=295 loss=4.222761 pg=4.222761 kl=0.000000 reward=0.577350 skipped=294 +[train] epoch=22 step=296 loss=6.120806 pg=6.120743 kl=0.006336 reward=1.000000 skipped=295 +[train] epoch=22 step=297 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=296 +[train] epoch=22 step=298 loss=0.000013 pg=0.000000 kl=0.001251 reward=0.000000 skipped=297 +[train] epoch=22 step=299 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=298 +[train] epoch=22 step=300 loss=0.000010 pg=0.000000 kl=0.000967 reward=0.000000 skipped=299 +[train] epoch=22 step=301 loss=6.862856 pg=6.862832 kl=0.002382 reward=1.000000 skipped=300 +[train] epoch=22 step=302 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=301 +[train] epoch=22 step=303 loss=5.151323 pg=5.151323 kl=0.000000 reward=1.000000 skipped=302 +[train] epoch=22 step=304 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=303 +[train] epoch=22 step=305 loss=7.065364 pg=7.065358 kl=0.000586 reward=1.000000 skipped=304 +[train] epoch=22 step=306 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=305 +[train] epoch=22 step=307 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=306 +[train] epoch=22 step=308 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=307 +[train] epoch=22 step=309 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=308 +[train] epoch=22 step=310 loss=4.706718 pg=4.706718 kl=0.000000 reward=1.000000 skipped=309 +[train] epoch=22 step=311 loss=6.261615 pg=6.261593 kl=0.002169 reward=1.000000 skipped=310 +[train] epoch=22 step=312 loss=4.211183 pg=4.211183 kl=0.000000 reward=0.774597 skipped=311 +[train] epoch=22 step=313 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=312 +[train] epoch=22 step=314 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=313 +[train] epoch=22 step=315 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=314 +[train] epoch=22 step=316 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=315 +[train] epoch=22 step=317 loss=5.321369 pg=5.321357 kl=0.001224 reward=1.000000 skipped=316 +[train] epoch=22 step=318 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=317 +[train] epoch=22 step=319 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=318 +[train] epoch=22 step=320 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=319 +[train] epoch=22 step=321 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=320 +[train] epoch=22 step=322 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=321 +[train] epoch=22 step=323 loss=7.256117 pg=7.256107 kl=0.000982 reward=1.000000 skipped=322 +[train] epoch=22 step=324 loss=6.963132 pg=6.963132 kl=0.000000 reward=1.000000 skipped=323 +[train] epoch=22 step=325 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=324 +[train] epoch=22 step=326 loss=5.592641 pg=5.592641 kl=0.000000 reward=1.000000 skipped=325 +[train] epoch=22 step=327 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=326 +[train] epoch=22 step=328 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=327 +[train] epoch=22 step=329 loss=5.369410 pg=5.369405 kl=0.000436 reward=1.000000 skipped=328 +[train] epoch=22 step=330 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=329 +[train] epoch=22 step=331 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=330 +[train] epoch=22 step=332 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=331 +[train] epoch=22 step=333 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=332 +[train] epoch=22 step=334 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=333 +[train] epoch=22 step=335 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=334 +[train] epoch=22 step=336 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=335 +[train] epoch=22 step=337 loss=5.369410 pg=5.369405 kl=0.000436 reward=1.000000 skipped=336 +[train] epoch=22 step=338 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=337 +[train] epoch=22 step=339 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=338 +[train] epoch=22 step=340 loss=14.783320 pg=14.783320 kl=0.000000 reward=2.645751 skipped=339 +[train] epoch=22 step=341 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=340 +[train] epoch=22 step=342 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=341 +[train] epoch=22 step=343 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=342 +[train] epoch=22 step=344 loss=2.389125 pg=2.389019 kl=0.010641 reward=0.377964 skipped=343 +[train] epoch=22 step=345 loss=4.665239 pg=4.665239 kl=0.000000 reward=1.000000 skipped=344 +[train] epoch=22 step=346 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=345 +[train] epoch=22 step=347 loss=6.593108 pg=6.593024 kl=0.008324 reward=1.000000 skipped=346 +[train] epoch=22 step=348 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=347 +[train] epoch=22 step=349 loss=5.696458 pg=5.696399 kl=0.005970 reward=1.000000 skipped=348 +[train] epoch=22 step=350 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=349 +[train] epoch=22 step=351 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=350 +[train] epoch=22 step=352 loss=7.021185 pg=7.021152 kl=0.003361 reward=1.000000 skipped=351 +[train] epoch=22 step=353 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=352 +[train] epoch=22 step=354 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=353 +[train] epoch=22 step=355 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=354 +[train] epoch=22 step=356 loss=2.249304 pg=2.249304 kl=0.000000 reward=0.377964 skipped=355 +[train] epoch=22 step=357 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=356 +[train] epoch=22 step=358 loss=8.118921 pg=8.118921 kl=0.000000 reward=1.000000 skipped=357 +[train] epoch=22 step=359 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=358 +[train] epoch=22 step=360 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=359 +[train] epoch=22 step=361 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=360 +[train] epoch=22 step=362 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=361 +[train] epoch=22 step=363 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=362 +[train] epoch=22 step=364 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=363 +[train] epoch=22 step=365 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=364 +[train] epoch=22 step=366 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=365 +[train] epoch=22 step=367 loss=-8.813005 pg=-8.813011 kl=0.000536 reward=-1.290994 skipped=366 +[train] epoch=22 step=368 loss=5.696458 pg=5.696399 kl=0.005970 reward=1.000000 skipped=367 +[train] epoch=22 step=369 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=368 +[train] epoch=22 step=370 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=369 +[train] epoch=22 step=371 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=370 +[train] epoch=22 step=372 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=371 +[train] epoch=22 step=373 loss=7.845365 pg=7.845365 kl=0.000000 reward=1.000000 skipped=372 +[train] epoch=22 step=374 loss=6.427264 pg=6.427164 kl=0.010050 reward=1.000000 skipped=373 +[train] epoch=22 step=375 loss=8.245316 pg=8.245316 kl=0.000000 reward=1.000000 skipped=374 +[train] epoch=22 step=376 loss=5.937304 pg=5.937304 kl=0.000000 reward=1.000000 skipped=375 +[train] epoch=22 step=377 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=376 +[train] epoch=22 step=378 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=377 +[train] epoch=22 step=379 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=378 +[train] epoch=22 step=380 loss=-17.326363 pg=-17.326363 kl=0.000000 reward=-2.645751 skipped=379 +[train] epoch=22 step=381 loss=6.194232 pg=6.194232 kl=0.000000 reward=1.000000 skipped=380 +[train] epoch=22 step=382 loss=5.951652 pg=5.951652 kl=0.000000 reward=1.000000 skipped=381 +[train] epoch=22 step=383 loss=6.194232 pg=6.194232 kl=0.000000 reward=1.000000 skipped=382 +[train] epoch=22 step=384 loss=6.548525 pg=6.548453 kl=0.007184 reward=1.000000 skipped=383 +[train] epoch=22 step=385 loss=4.836777 pg=4.836777 kl=0.000000 reward=1.000000 skipped=384 +[train] epoch=22 step=386 loss=4.694529 pg=4.694529 kl=0.000000 reward=1.000000 skipped=385 +[train] epoch=22 step=387 loss=7.204127 pg=7.204088 kl=0.003951 reward=1.000000 skipped=386 +[train] epoch=22 step=388 loss=8.060417 pg=8.060417 kl=0.000000 reward=1.000000 skipped=387 +[train] epoch=22 step=389 loss=5.734613 pg=5.734613 kl=0.000000 reward=1.000000 skipped=388 +[train] epoch=22 step=390 loss=6.017231 pg=6.017231 kl=0.000000 reward=1.000000 skipped=389 +[train] epoch=22 step=391 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=390 +[train] epoch=22 step=392 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=391 +[train] epoch=22 step=393 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=392 +[train] epoch=22 step=394 loss=6.292162 pg=6.292067 kl=0.009594 reward=1.000000 skipped=393 +[train] epoch=22 step=395 loss=5.446908 pg=5.446908 kl=0.000000 reward=1.000000 skipped=394 +[train] epoch=22 step=396 loss=6.173695 pg=6.173695 kl=0.000000 reward=1.000000 skipped=395 +[train] epoch=22 step=397 loss=-14.919147 pg=-14.919180 kl=0.003295 reward=-2.645751 skipped=396 +[train] epoch=22 step=398 loss=6.337766 pg=6.337766 kl=0.000000 reward=1.000000 skipped=397 +[train] epoch=22 step=399 loss=6.197713 pg=6.197596 kl=0.011770 reward=1.000000 skipped=398 +[train] epoch=22 step=400 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=399 +[train] epoch=22 step=401 loss=5.734613 pg=5.734613 kl=0.000000 reward=1.000000 skipped=400 +[train] epoch=22 step=402 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=401 +[train] epoch=22 step=403 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=402 +[train] epoch=22 step=404 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=403 +[train] epoch=22 step=405 loss=6.120806 pg=6.120743 kl=0.006336 reward=1.000000 skipped=404 +[train] epoch=22 step=406 loss=6.954225 pg=6.954225 kl=0.000000 reward=1.000000 skipped=405 +[train] epoch=22 step=407 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=406 +[train] epoch=22 step=408 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=407 +[train] epoch=22 step=409 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=408 +[train] epoch=22 step=410 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=409 +[train] epoch=22 step=411 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=410 +[train] epoch=22 step=412 loss=6.578031 pg=6.578004 kl=0.002699 reward=1.000000 skipped=411 +[train] epoch=22 step=413 loss=6.387280 pg=6.387263 kl=0.001739 reward=1.000000 skipped=412 +[train] epoch=22 step=414 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=413 +[train] epoch=22 step=415 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=414 +[train] epoch=22 step=416 loss=6.134849 pg=6.134849 kl=0.000000 reward=1.000000 skipped=415 +[train] epoch=22 step=417 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=416 +[train] epoch=22 step=418 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=417 +[train] epoch=22 step=419 loss=2.614115 pg=2.614115 kl=0.000000 reward=0.377964 skipped=418 +[train] epoch=22 step=420 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=419 +[train] epoch=22 step=421 loss=0.000010 pg=0.000000 kl=0.000967 reward=0.000000 skipped=420 +[train] epoch=22 step=422 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=421 +[train] epoch=22 step=423 loss=7.200075 pg=7.200075 kl=0.000000 reward=1.000000 skipped=422 +[train] epoch=22 step=424 loss=6.746428 pg=6.746428 kl=0.000000 reward=1.000000 skipped=423 +[train] epoch=22 step=425 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=424 +[train] epoch=22 step=426 loss=-18.112778 pg=-18.112778 kl=0.000000 reward=-2.645751 skipped=425 +[train] epoch=22 step=427 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=426 +[train] epoch=22 step=428 loss=5.438523 pg=5.438523 kl=0.000000 reward=1.000000 skipped=427 +[train] epoch=22 step=429 loss=6.576011 pg=6.575956 kl=0.005465 reward=1.000000 skipped=428 +[train] epoch=22 step=430 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=429 +[train] epoch=22 step=431 loss=0.000035 pg=0.000000 kl=0.003548 reward=0.000000 skipped=430 +[train] epoch=22 step=432 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=431 +[train] epoch=22 step=433 loss=8.089325 pg=8.089325 kl=0.000000 reward=1.000000 skipped=432 +[train] epoch=22 step=434 loss=4.886762 pg=4.886762 kl=0.000000 reward=1.000000 skipped=433 +[train] epoch=22 step=435 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=434 +[train] epoch=22 step=436 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=435 +[train] epoch=22 step=437 loss=6.225890 pg=6.225890 kl=0.000000 reward=1.000000 skipped=436 +[train] epoch=22 step=438 loss=4.680637 pg=4.680637 kl=0.000000 reward=1.000000 skipped=437 +[train] epoch=22 step=439 loss=7.161344 pg=7.161344 kl=0.000000 reward=1.000000 skipped=438 +[train] epoch=22 step=440 loss=7.312311 pg=7.312311 kl=0.000000 reward=1.000000 skipped=439 +[train] epoch=22 step=441 loss=5.665410 pg=5.665351 kl=0.005796 reward=1.000000 skipped=440 +[train] epoch=22 step=442 loss=6.276326 pg=6.276241 kl=0.008499 reward=1.000000 skipped=441 +[train] epoch=22 step=443 loss=7.204127 pg=7.204088 kl=0.003951 reward=1.000000 skipped=442 +[train] epoch=22 step=444 loss=7.282161 pg=7.282101 kl=0.006026 reward=1.000000 skipped=443 +[train] epoch=22 step=445 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=444 +[train] epoch=22 step=446 loss=5.696458 pg=5.696399 kl=0.005970 reward=1.000000 skipped=445 +[train] epoch=22 step=447 loss=6.332105 pg=6.332021 kl=0.008396 reward=1.000000 skipped=446 +[train] epoch=22 step=448 loss=6.735466 pg=6.735466 kl=0.000000 reward=1.000000 skipped=447 +[train] epoch=22 step=449 loss=8.072202 pg=8.072189 kl=0.001261 reward=1.000000 skipped=448 +[train] epoch=22 step=450 loss=6.703228 pg=6.703189 kl=0.003809 reward=1.000000 skipped=449 +[train] epoch=22 step=451 loss=7.820656 pg=7.820656 kl=0.000000 reward=1.000000 skipped=450 +[train] epoch=22 step=452 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=451 +[train] epoch=22 step=453 loss=8.217723 pg=8.217719 kl=0.000403 reward=1.000000 skipped=452 +[train] epoch=22 step=454 loss=5.819166 pg=5.819166 kl=0.000000 reward=1.000000 skipped=453 +[train] epoch=22 step=455 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=454 +[train] epoch=22 step=456 loss=6.115604 pg=6.115604 kl=0.000000 reward=1.000000 skipped=455 +[train] epoch=22 step=457 loss=5.766104 pg=5.766104 kl=0.000000 reward=1.000000 skipped=456 +[train] epoch=22 step=458 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=457 +[train] epoch=22 step=459 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=458 +[train] epoch=22 step=460 loss=6.708456 pg=6.708344 kl=0.011145 reward=1.000000 skipped=459 +[train] epoch=22 step=461 loss=2.614115 pg=2.614115 kl=0.000000 reward=0.377964 skipped=460 +[train] epoch=22 step=462 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=461 +[train] epoch=22 step=463 loss=7.312311 pg=7.312311 kl=0.000000 reward=1.000000 skipped=462 +[train] epoch=22 step=464 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=463 +[train] epoch=22 step=465 loss=5.781769 pg=5.781659 kl=0.011064 reward=1.000000 skipped=464 +[train] epoch=22 step=466 loss=6.080158 pg=6.080158 kl=0.000000 reward=1.000000 skipped=465 +[train] epoch=22 step=467 loss=6.071664 pg=6.071613 kl=0.005109 reward=1.000000 skipped=466 +[train] epoch=22 step=468 loss=5.674278 pg=5.674278 kl=0.000000 reward=1.000000 skipped=467 +[train] epoch=22 step=469 loss=5.792642 pg=5.792642 kl=0.000000 reward=1.000000 skipped=468 +[train] epoch=22 step=470 loss=6.301637 pg=6.301637 kl=0.000000 reward=1.000000 skipped=469 +[train] epoch=22 step=471 loss=8.245316 pg=8.245316 kl=0.000000 reward=1.000000 skipped=470 +[train] epoch=22 step=472 loss=6.746428 pg=6.746428 kl=0.000000 reward=1.000000 skipped=471 +[train] epoch=22 step=473 loss=5.360440 pg=5.360439 kl=0.000106 reward=1.000000 skipped=472 +[train] epoch=22 step=474 loss=7.096770 pg=7.096770 kl=0.000000 reward=1.000000 skipped=473 +[train] epoch=22 step=475 loss=0.000013 pg=0.000000 kl=0.001251 reward=0.000000 skipped=474 +[train] epoch=22 step=476 loss=5.151323 pg=5.151323 kl=0.000000 reward=1.000000 skipped=475 +[train] epoch=22 step=477 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=476 +[train] epoch=22 step=478 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=477 +[train] epoch=22 step=479 loss=7.880197 pg=7.880185 kl=0.001245 reward=1.000000 skipped=478 +[train] epoch=22 step=480 loss=7.098940 pg=7.098918 kl=0.002206 reward=1.000000 skipped=479 +[train] epoch=22 step=481 loss=6.000207 pg=6.000207 kl=0.000000 reward=1.000000 skipped=480 +[train] epoch=22 step=482 loss=7.793433 pg=7.793433 kl=0.000000 reward=1.000000 skipped=481 +[train] epoch=22 step=483 loss=5.568174 pg=5.568095 kl=0.007986 reward=1.000000 skipped=482 +[train] epoch=22 step=484 loss=0.000000 pg=0.000000 kl=0.000000 reward=0.000000 skipped=483 +[train] epoch=22 step=485 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=484 +[train] epoch=22 step=486 loss=6.319226 pg=6.319215 kl=0.001165 reward=1.000000 skipped=485 +[train] epoch=22 step=487 loss=5.581098 pg=5.581098 kl=0.000000 reward=1.000000 skipped=486 +[train] epoch=22 step=488 loss=5.581098 pg=5.581098 kl=0.000000 reward=1.000000 skipped=487 +[train] epoch=22 step=489 loss=6.484924 pg=6.484878 kl=0.004605 reward=1.000000 skipped=488 +[train] epoch=22 step=490 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=489 +[train] epoch=22 step=491 loss=4.640452 pg=4.640452 kl=0.000000 reward=1.000000 skipped=490 +[train] epoch=22 step=492 loss=6.627009 pg=6.626989 kl=0.002073 reward=1.000000 skipped=491 +[train] epoch=22 step=493 loss=6.847517 pg=6.847517 kl=0.000000 reward=1.000000 skipped=492 +[train] epoch=22 step=494 loss=5.151323 pg=5.151323 kl=0.000000 reward=1.000000 skipped=493 +[train] epoch=22 step=495 loss=7.204127 pg=7.204088 kl=0.003951 reward=1.000000 skipped=494 +[train] epoch=22 step=496 loss=5.476670 pg=5.476632 kl=0.003761 reward=1.000000 skipped=495 +[train] epoch=22 step=497 loss=4.222761 pg=4.222761 kl=0.000000 reward=0.577350 skipped=496 +[train] epoch=22 step=498 loss=2.249304 pg=2.249304 kl=0.000000 reward=0.377964 skipped=497 +[train] epoch=22 step=499 loss=4.794056 pg=4.794045 kl=0.001111 reward=1.000000 skipped=498 +[train] epoch=22 step=500 loss=-10.394114 pg=-10.394114 kl=0.000000 reward=-1.732051 skipped=499 +[train] epoch=22 step=501 loss=5.432666 pg=5.432619 kl=0.004674 reward=1.000000 skipped=500 +[train] epoch=22 step=502 loss=5.585279 pg=5.585279 kl=0.000000 reward=1.000000 skipped=501 +[train] epoch=22 step=503 loss=5.875602 pg=5.875589 kl=0.001239 reward=1.000000 skipped=502 +[train] epoch=22 step=504 loss=6.147610 pg=6.147610 kl=0.000000 reward=1.000000 skipped=503 +[train] epoch=22 step=505 loss=4.222761 pg=4.222761 kl=0.000000 reward=0.577350 skipped=504 +[train] epoch=22 step=506 loss=7.282108 pg=7.282108 kl=0.000000 reward=1.000000 skipped=505 +[train] epoch=22 step=507 loss=6.783085 pg=6.783085 kl=0.000000 reward=1.000000 skipped=506 +[train] epoch=22 step=508 loss=5.614491 pg=5.614491 kl=0.000000 reward=1.000000 skipped=507 +[train] epoch=22 step=509 loss=5.282966 pg=5.282931 kl=0.003500 reward=1.000000 skipped=508 +[train] epoch=22 step=510 loss=5.964911 pg=5.964870 kl=0.004068 reward=1.000000 skipped=509 +[train] epoch=22 step=511 loss=6.470130 pg=6.470130 kl=0.000000 reward=1.000000 skipped=510 +[train] epoch=22 step=512 loss=8.032864 pg=8.032864 kl=0.000000 reward=1.000000 skipped=511 +[train] epoch=22 checkpoint_saved=/data4/runqing/dLLM-RL/rl_llava_next_video_reversal/ckpt/optimized diff --git a/wandb/run-20260601_170919-9x12ylug/files/requirements.txt b/wandb/run-20260601_170919-9x12ylug/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..75f723721d959cc9661b3ed252c3052267b34822 --- /dev/null +++ b/wandb/run-20260601_170919-9x12ylug/files/requirements.txt @@ -0,0 +1,167 @@ +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +tokenizers==0.15.2 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +markdown-it-py==4.2.0 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +accelerate==1.13.0 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +six==1.17.0 +Pygments==2.19.2 +open_clip_torch==3.3.0 +transformers==4.40.0.dev0 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +mdurl==0.1.2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +rich==15.0.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +setuptools==80.10.2 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +peft==0.4.0 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +aiosignal==1.4.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +shtab==1.8.0 +opencv-python-headless==4.9.0.80 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +docstring_parser==0.18.0 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +tyro==0.8.3 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +ftfy==6.3.1 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +timm==1.0.27 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260601_170919-9x12ylug/files/wandb-metadata.json b/wandb/run-20260601_170919-9x12ylug/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3f70af4c2ec1ce09b0dc783fb7b3d062a472658f --- /dev/null +++ b/wandb/run-20260601_170919-9x12ylug/files/wandb-metadata.json @@ -0,0 +1,94 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-01T09:09:19.545899Z", + "args": [ + "config=configs/rl_llava_next_video_reversal.yaml", + "wandb.run_id=9x12ylug", + "experiment.current_epoch=22" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_llava_next_video.py", + "codePath": "train/rl_llava_next_video.py", + "codePathLocal": "train/rl_llava_next_video.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/llava-next-video/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "626407727104" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "63ljj4arc5l9774b36d31nxbetbv5xgb" +} \ No newline at end of file diff --git a/wandb/run-20260601_170919-9x12ylug/files/wandb-summary.json b/wandb/run-20260601_170919-9x12ylug/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e017bbfe189db0d24948a3c5718cf001f91770 --- /dev/null +++ b/wandb/run-20260601_170919-9x12ylug/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/advantage":1,"train/skipped_steps":511,"_wandb":{"runtime":13153},"_step":11263,"train/loss":8.03286361694336,"train/pg_loss":8.03286361694336,"_timestamp":1.7803055014134793e+09,"train/step":512,"_runtime":13153,"train/kl_loss":0} \ No newline at end of file diff --git a/wandb/run-20260601_170919-9x12ylug/logs/debug-core.log b/wandb/run-20260601_170919-9x12ylug/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b205a2c15d68cb35bd57d8503f18d5197bee6369 --- /dev/null +++ b/wandb/run-20260601_170919-9x12ylug/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-06-01T17:09:19.614750774+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp9dkyjr4v/port-2578376.txt","pid":2578376,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-01T17:09:19.615760698+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2578376} +{"time":"2026-06-01T17:09:19.615746603+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2578376-2578752-4165679331/socket","Net":"unix"}} +{"time":"2026-06-01T17:09:19.794974897+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-01T17:09:19.805442036+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T17:09:20.108367612+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T17:09:25.451811744+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"2kehfe1mxlpm"} +{"time":"2026-06-01T17:19:08.286929771+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"2kehfe1mxlpm"} +{"time":"2026-06-01T17:19:08.541075744+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"2kehfe1mxlpm"} +{"time":"2026-06-01T17:19:08.541558356+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T17:19:08.542631868+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"9x12ylug","id":"1(@)"} +{"time":"2026-06-01T17:19:08.558860365+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-01T17:19:08.558905651+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-01T17:19:08.558916578+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-01T17:19:08.55893026+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-01T17:19:08.559056259+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2578376-2578752-4165679331/socket","Net":"unix"}} +{"time":"2026-06-01T17:19:08.558955499+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-01T17:19:08.559122907+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-01T17:19:08.559130038+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260601_170919-9x12ylug/logs/debug-internal.log b/wandb/run-20260601_170919-9x12ylug/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b8003e6f75a37ec0dabdc19476c703a21bacac9f --- /dev/null +++ b/wandb/run-20260601_170919-9x12ylug/logs/debug-internal.log @@ -0,0 +1,30 @@ +{"time":"2026-06-01T17:09:19.80559905+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-01T17:09:20.107605763+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-01T17:09:20.107977981+08:00","level":"INFO","msg":"stream: created new stream","id":"9x12ylug"} +{"time":"2026-06-01T17:09:20.10835421+08:00","level":"INFO","msg":"stream: started","id":"9x12ylug"} +{"time":"2026-06-01T17:09:20.108938733+08:00","level":"INFO","msg":"handler: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T17:09:20.10900242+08:00","level":"INFO","msg":"sender: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T17:09:20.108936655+08:00","level":"INFO","msg":"writer: started","stream_id":"9x12ylug"} +{"time":"2026-06-01T17:09:20.111287826+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-01T17:09:25.407930323+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T090925Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0788a8876a9665198e1fe937290bb4c3ef18e06c48fd64af69f3e30a2c84ee8c"} +{"time":"2026-06-01T17:09:25.408032161+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txtnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt18B4E9EEFDFE24AD4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_170919-9x12ylug/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T090925Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0788a8876a9665198e1fe937290bb4c3ef18e06c48fd64af69f3e30a2c84ee8c, Size: 3167}"} +{"time":"2026-06-01T17:09:25.409231528+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T090925Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6c58c1ebcc386e500be59a308d629fe85a1510ab05ad66542a4ee66855650b63"} +{"time":"2026-06-01T17:09:25.409302383+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json18B4E9EEFD915C3417e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_170919-9x12ylug/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T090925Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6c58c1ebcc386e500be59a308d629fe85a1510ab05ad66542a4ee66855650b63, Size: 2704}"} +{"time":"2026-06-01T17:09:25.417420303+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T090925Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e0286e642f678aa7f6614d99c24c17ac8da5edcca61d91266d87f71d95aa902a"} +{"time":"2026-06-01T17:09:25.417510518+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18B4E9EEFE23E0554b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_170919-9x12ylug/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T090925Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e0286e642f678aa7f6614d99c24c17ac8da5edcca61d91266d87f71d95aa902a, Size: 14311}"} +{"time":"2026-06-01T17:09:25.421045996+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T090925Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2d6b1f16202636b41bbe7d845aca40cee5edce86fc5692a062cc01ab72a7a37b"} +{"time":"2026-06-01T17:09:25.421153838+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patchnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch18B4E9EEFE33B71B4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_170919-9x12ylug/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T090925Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2d6b1f16202636b41bbe7d845aca40cee5edce86fc5692a062cc01ab72a7a37b, Size: 14311}"} +{"time":"2026-06-01T17:09:25.42447581+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T090925Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c614fcae5353e4e04e60e01567b2e38dbfe53d34c8fa07e45cdcf9af40dc8458"} +{"time":"2026-06-01T17:09:25.424553809+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.pynetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py18B4E9EEFFAC386929678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_170919-9x12ylug/files/code/train/rl_llava_next_video.py, Name: code/train/rl_llava_next_video.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/code/train/rl_llava_next_video.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T090925Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c614fcae5353e4e04e60e01567b2e38dbfe53d34c8fa07e45cdcf9af40dc8458, Size: 15437}"} +{"time":"2026-06-01T17:19:08.415291522+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T091908Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce35a51c84fe1959bed79ba421a49f2d36dd3b29ea23cc5d57e68610be32b723"} +{"time":"2026-06-01T17:19:08.415372351+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json18B4EA76BD20E7DB29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_170919-9x12ylug/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T091908Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce35a51c84fe1959bed79ba421a49f2d36dd3b29ea23cc5d57e68610be32b723, Size: 239}"} +{"time":"2026-06-01T17:19:08.430870672+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T091908Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1b792ada1a4d02d5ee2085022443badb6d72881808ce3c7f1e2f4fdfdd75681e"} +{"time":"2026-06-01T17:19:08.430945623+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.lognetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log18B4EA76BD04C89229678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_170919-9x12ylug/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T091908Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1b792ada1a4d02d5ee2085022443badb6d72881808ce3c7f1e2f4fdfdd75681e, Size: 48868}"} +{"time":"2026-06-01T17:19:08.491979165+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T091908Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=218dfbd61b118088d3187c8487cf54b7af0f482ebd295ae8a00ab204dddffe89"} +{"time":"2026-06-01T17:19:08.492116982+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yamlnetdisk/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml18B4EA76C0020D7717e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260601_170919-9x12ylug/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_llava_next_video_reversal/9x12ylug/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260601%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260601T091908Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=218dfbd61b118088d3187c8487cf54b7af0f482ebd295ae8a00ab204dddffe89, Size: 7149}"} +{"time":"2026-06-01T17:19:08.492534111+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-01T17:19:08.535254453+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-01T17:19:08.541570699+08:00","level":"INFO","msg":"stream: closing","id":"9x12ylug"} +{"time":"2026-06-01T17:19:08.541587743+08:00","level":"INFO","msg":"handler: closed","stream_id":"9x12ylug"} +{"time":"2026-06-01T17:19:08.541664141+08:00","level":"INFO","msg":"sender: closed","stream_id":"9x12ylug"} +{"time":"2026-06-01T17:19:08.541674085+08:00","level":"INFO","msg":"stream: closed","id":"9x12ylug"} diff --git a/wandb/run-20260601_170919-9x12ylug/logs/debug.log b/wandb/run-20260601_170919-9x12ylug/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2b2f64f70a944a9a81e5bd9f224fe09849693f4d --- /dev/null +++ b/wandb/run-20260601_170919-9x12ylug/logs/debug.log @@ -0,0 +1,25 @@ +2026-06-01 17:09:19,548 INFO MainThread:2578376 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-01 17:09:19,549 INFO MainThread:2578376 [wandb_setup.py:_flush():81] Configure stats pid to 2578376 +2026-06-01 17:09:19,549 INFO MainThread:2578376 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-01 17:09:19,549 INFO MainThread:2578376 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260601_170919-9x12ylug/logs/debug.log +2026-06-01 17:09:19,549 INFO MainThread:2578376 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260601_170919-9x12ylug/logs/debug-internal.log +2026-06-01 17:09:19,549 INFO MainThread:2578376 [wandb_init.py:init():844] calling init triggers +2026-06-01 17:09:19,549 INFO MainThread:2578376 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': '9x12ylug', 'system.env_name': '/data8/runqing/miniconda/envs/llava-next-video', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_llava_next_video_reversal', 'experiment.function': 'train', 'experiment.current_epoch': 22, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.reference_model': '/data4/runqing/dLLM-RL/sft_llava_next_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_name': 'llava_llama', 'model.model_base': None, 'model.conv_mode': 'vicuna_v1', 'model.load_overwrite_config.mm_vision_tower': '/data4/runqing/hf/clip-vit-large-patch14-336', 'model.load_overwrite_config.mm_spatial_pool_stride': 2, 'model.load_overwrite_config.mm_spatial_pool_mode': 'average', 'model.load_overwrite_config.mm_newline_position': 'no_token', 'model.load_overwrite_config.max_sequence_length': 8192, 'model.load_overwrite_config.tokenizer_model_max_length': 8192, 'model.load_overwrite_config.rope_scaling.factor': 2.0, 'model.load_overwrite_config.rope_scaling.type': 'linear', 'model.load_overwrite_config.video_fps': 1, 'model.load_overwrite_config.frames_upbound': 32, 'model.load_overwrite_config.force_sample': True, 'model.load_overwrite_config.add_time_instruction': True, 'dataset.train_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_train.json', 'dataset.optimization_data_file': 'rl_data.json', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.top_p': 0.95, 'rollout.max_new_tokens': 128, 'reward.max_direction_gap': 0.2, 'reward.direction_consistency_bonus': 0.3, 'training.num_train_epochs': 1, 'training.gradient_accumulation_steps': 2, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.max_grad_norm': 1.0, 'training.kl_beta': 0.01, 'training.logging_steps': 1, 'training.trainable_parts.0': 'mm_projector', 'training.load_overwrite_config.max_sequence_length': 2048, 'training.load_overwrite_config.tokenizer_model_max_length': 2048, 'training.load_overwrite_config.frames_upbound': 8, 'optimizer.learning_rate': 1e-06, 'optimizer.weight_decay': 0.0, 'evaluation.eval_json': '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json', 'evaluation.num_task_per_step': 64, 'evaluation.num_response_per_task': 3, 'evaluation.max_new_tokens': 128, 'evaluation.temperature': 0.1, 'config': 'configs/rl_llava_next_video_reversal.yaml', '_wandb': {'code_path': 'code/train/rl_llava_next_video.py'}} +2026-06-01 17:09:19,549 INFO MainThread:2578376 [wandb_init.py:init():892] starting backend +2026-06-01 17:09:19,795 INFO MainThread:2578376 [wandb_init.py:init():895] sending inform_init request +2026-06-01 17:09:19,801 INFO MainThread:2578376 [wandb_init.py:init():903] backend started and connected +2026-06-01 17:09:19,803 INFO MainThread:2578376 [wandb_init.py:init():973] updated telemetry +2026-06-01 17:09:19,809 INFO MainThread:2578376 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-01 17:09:20,332 INFO MainThread:2578376 [wandb_init.py:init():1037] run resumed +2026-06-01 17:09:20,337 INFO MainThread:2578376 [wandb_init.py:init():1042] starting run threads in backend +2026-06-01 17:09:20,448 INFO MainThread:2578376 [wandb_run.py:_console_start():2524] atexit reg +2026-06-01 17:09:20,448 INFO MainThread:2578376 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-01 17:09:20,448 INFO MainThread:2578376 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-01 17:09:20,448 INFO MainThread:2578376 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-01 17:09:20,452 INFO MainThread:2578376 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-01 17:19:08,275 INFO MainThread:2578376 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_llava_next_video_reversal/9x12ylug +2026-06-01 17:19:08,276 INFO MainThread:2578376 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-01 17:19:08,286 INFO MainThread:2578376 [wandb_run.py:_restore():2472] restore +2026-06-01 17:19:08,286 INFO MainThread:2578376 [wandb_run.py:_restore():2478] restore done +2026-06-01 17:19:08,540 INFO MainThread:2578376 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/wandb/run-20260602_132409-w3wdpl8c/files/code/train/rl_lavida.py b/wandb/run-20260602_132409-w3wdpl8c/files/code/train/rl_lavida.py new file mode 100644 index 0000000000000000000000000000000000000000..3e111f99f33455c8199e45d6e289cc4dfee34b6d --- /dev/null +++ b/wandb/run-20260602_132409-w3wdpl8c/files/code/train/rl_lavida.py @@ -0,0 +1,840 @@ +import os +import sys +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LAVIDA_ROOT = os.path.join(REPO_ROOT, "LaViDa") +sys.path.insert(0, LAVIDA_ROOT) +sys.path.insert(1, REPO_ROOT) +os.environ["TOKENIZERS_PARALLELISM"] = "true" +from datetime import timedelta +import json +import logging +import math +import shutil +import time +from pathlib import Path +from typing import Dict, List, Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.utils.data import Dataset, DataLoader + +from omegaconf import OmegaConf +import wandb + +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed + +from train.utils import get_config, flatten_omega_conf, AverageMeter, ensure_wandb_run_id +from models.lr_schedulers import get_scheduler +from models.logging import set_verbosity_info, set_verbosity_error + +from transformers.utils import logging as hf_logging +hf_logging.set_verbosity_error() + +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, tokenizer_image_token, process_video_with_decord +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates + +from termcolor import cprint + +logger = get_logger(__name__, log_level="INFO") + +LAVIDA_MODEL_NAME = "llava_llada" + + +def collapse_k_unique(lst, k: int): + if k <= 0: + raise ValueError("k must be > 0") + uniq = sorted(set(lst)) + mapping = {} + n = len(uniq) + for idx, val in enumerate(uniq): + group = idx // k + end_idx = min((group + 1) * k - 1, n - 1) + rep = uniq[end_idx] + mapping[val] = rep + return [mapping[x] for x in lst] + + +def build_llava_conv_prompt(prompt_text: str, response_text: str) -> str: + import copy + conv = copy.deepcopy(conv_templates["llada"]) + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + prompt_text) + conv.append_message(conv.roles[1], response_text) + return conv.get_prompt().removesuffix("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") + + +def tokenize_conv_with_image(prompt_text: str, response_text: str, tokenizer) -> torch.Tensor: + conv_prompt = build_llava_conv_prompt(prompt_text, response_text) + ids = tokenizer_image_token(conv_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + return ids.squeeze(0) + + +def load_and_process_media(abs_path: str, media_type: str, image_processor, model_cfg, device, dtype=torch.bfloat16): + from PIL import Image + from types import SimpleNamespace + + if isinstance(model_cfg, dict): + model_cfg = SimpleNamespace(**model_cfg) + + if media_type == "image": + image = Image.open(abs_path).convert("RGB") + if not hasattr(model_cfg, "image_aspect_ratio") or getattr(model_cfg, "image_aspect_ratio") is None: + setattr(model_cfg, "image_aspect_ratio", "pad") + if not hasattr(model_cfg, "mm_patch_merge_type") or getattr(model_cfg, "mm_patch_merge_type") is None: + setattr(model_cfg, "mm_patch_merge_type", "flat") + + image_tensor_list = process_images([image], image_processor, model_cfg) + image_tensor_list = [_im.to(dtype=dtype, device=device) for _im in image_tensor_list] + image_sizes = [image.size] + return image_tensor_list, image_sizes, None + + elif media_type == "video": + if not hasattr(model_cfg, "video_fps"): setattr(model_cfg, "video_fps", 1) + if not hasattr(model_cfg, "frames_upbound"): setattr(model_cfg, "frames_upbound", 16) + if not hasattr(model_cfg, "force_sample"): setattr(model_cfg, "force_sample", True) + + video_frames, video_time, frame_time, num_frames_to_sample = process_video_with_decord(abs_path, model_cfg) + image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"] + image_tensor = image_tensor.to(dtype=dtype, device=device) + + if isinstance(video_frames, np.ndarray) and len(video_frames) > 0: + v_size = (video_frames.shape[2], video_frames.shape[1]) + else: + v_size = (0, 0) + + return [image_tensor], [v_size], (video_time, frame_time, num_frames_to_sample) + + +def left_pad_1d(x: torch.Tensor, L: int, pad_val: int): + out = torch.full((L,), pad_val, dtype=x.dtype, device=x.device) + out[-x.numel():] = x + return out + + +def left_pad_2d(x: torch.Tensor, L: int, pad_val: float = 0.0): + if x.size(0) == L: + return x + out = torch.full((L, x.size(1)), pad_val, dtype=x.dtype, device=x.device) + out[-x.size(0):] = x + return out + + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + + +class RLVariantDatasetLLADAV(Dataset): + def __init__(self, samples: List[dict], config): + super().__init__() + self.samples = samples + self.config = config + self.variants: List[dict] = [] + + # logp_old_tok mapping: idx -> tensor + self.logp_old_tok: Dict[int, torch.Tensor] = {} + + self._build_variants() + + def _build_variants(self): + sys_prompt = 'Carefully watch the video and pay attention to the cause and sequence of events, the detail and movement of objects, and the action and pose of persons. Based on your observations, select the best option that accurately addresses the question.\n' + question_prompt = "\nOnly give the best option." + multiple_choice_prompt = " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the tags." + + rng = np.random.default_rng(seed=self.config.training.seed or 42) + idx_counter = 0 + + for sid, s in enumerate(self.samples): + # Rollout produces "extracted_output" or "response", fall back if needed + response_text = s.get("response", s.get("extracted_output", "")) + if isinstance(response_text, list) and len(response_text) > 0: + response_text = response_text[0] # Just in case it's nested + + # If options exist (video multiple choice), construct full prompt + if "options" in s: + prompt_text = sys_prompt + 'Question: ' + s["prompt"] + 'Options: ' + str(s['options']) + question_prompt + multiple_choice_prompt + else: + prompt_text = s["prompt"] + + step_map = s.get("step_map", []) + + if self.config.training.method == "TraceRL": + order_list = collapse_k_unique(step_map, self.config.training.shrink) + uniq_steps = sorted(set(order_list)) + for st in uniq_steps: + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "TraceRL", int(st), order_list, None)) + idx_counter += 1 + + elif self.config.training.method == "random_masking": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "random_masking", t, None, mask_seed)) + idx_counter += 1 + + elif self.config.training.method == "coupled": + for _ in range(self.config.training.mask_times_per_sample): + t = float(rng.uniform(self.config.training.lower_p, self.config.training.upper_p)) + mask_seed = int(rng.integers(0, 2**31 - 1)) + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_rand", t, None, mask_seed)) + idx_counter += 1 + self.variants.append(self._create_var_dict(idx_counter, sid, prompt_text, response_text, s, "coupled_comp", t, None, mask_seed)) + idx_counter += 1 + else: + raise ValueError(f"Unknown RL method: {self.config.training.method}") + + def _create_var_dict(self, idx, sid, prompt, response, s, var_type, var_param, step_map, mask_seed): + return dict( + idx=idx, + sample_id=sid, + prompt=prompt, + response=response, + image_abs_path=s.get("image_abs_path", None), + video_abs_path=s.get("video_abs_path", None), + reward=s.get("reward", 0.0), + var_type=var_type, + var_param=var_param, + step_map=step_map, + mask_seed=mask_seed, + ) + + def __len__(self): + return len(self.variants) + + def __getitem__(self, idx): + return self.variants[idx] + + +def collate_variants(batch: List[dict]) -> Dict[str, List[Any]]: + out: Dict[str, List[Any]] = {} + keys = batch[0].keys() + for k in keys: + out[k] = [b[k] for b in batch] + return out + + +@torch.no_grad() +def prepare_batch_noisy_embeds_rl( + accelerator, + model, + tokenizer, + image_processor, + batch: Dict[str, List[Any]], + mask_id: int, + dataset_media_root: str | None, +) -> tuple | None: + + device = accelerator.device + bsz = len(batch["idx"]) + + group: Dict[int, List[int]] = {} + for i, sid in enumerate(batch["sample_id"]): + group.setdefault(int(sid), []).append(i) + + sample_cache: Dict[int, Dict[str, Any]] = {} + for sid, pos_list in group.items(): + first_pos = pos_list[0] + prompt_text = batch["prompt"][first_pos] + response_text = batch["response"][first_pos] + + sample_stub = { + "image_abs_path": batch.get("image_abs_path", [None])[first_pos], + "video_abs_path": batch.get("video_abs_path", [None])[first_pos], + } + + media_type = None + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="video") + if p_abs is not None: + media_type = "video" + else: + p_abs = resolve_media_abs_path(sample_stub, dataset_media_root, media_type="image") + if p_abs is not None: + media_type = "image" + + img_list, image_sizes, modalities = None, None, [] + if p_abs is not None: + img_list, image_sizes, video_stats = load_and_process_media( + p_abs, media_type, image_processor, model.config, device, dtype=next(model.parameters()).dtype + ) + modalities = [media_type] + + if media_type == "video" and getattr(model.config, "add_time_instruction", False): + video_time, frame_time, num_frames = video_stats + time_instruction = f"The video lasts for {video_time:.2f} seconds, and {num_frames} frames are uniformly sampled from it. These frames are located at {frame_time}. Please answer the following questions related to this video." + clean_prompt = prompt_text.replace(DEFAULT_IMAGE_TOKEN, "").strip() + prompt_text = f"{time_instruction}\n{clean_prompt}" + + input_ids_text = tokenize_conv_with_image(prompt_text, response_text, tokenizer).to(device) + end_header_seq = tokenizer.encode("<|end_header_id|>\n\n", add_special_tokens=False) + seq_len = len(end_header_seq) + pos_eh = -1 + if seq_len > 0 and input_ids_text.numel() >= seq_len: + ids_list = input_ids_text.tolist() + for i in range(len(ids_list) - seq_len, -1, -1): + if ids_list[i:i + seq_len] == end_header_seq: + pos_eh = i + seq_len - 1 + break + assert pos_eh != -1, " <|end_header_id|> not found" + start_pos = pos_eh + 1 + + labels_text = input_ids_text.clone() + labels_text[:start_pos] = -100 + + (_in_ids, _pos_ids, _attn_mask, _pkv, clean_embeds, new_labels) = model.prepare_inputs_labels_for_multimodal( + input_ids_text.unsqueeze(0), None, None, None, labels_text.unsqueeze(0), img_list, modalities, image_sizes=image_sizes + ) + clean_embeds = clean_embeds.squeeze(0) + new_labels = new_labels.squeeze(0) + + eot_id = 126348 + eot_indices = (new_labels == eot_id).nonzero(as_tuple=False) + if eot_indices.numel() > 0: + last_eot_pos = eot_indices[-1, 0] + new_labels[last_eot_pos:] = -100 + asst_mask = (new_labels != -100) + asst_idx = asst_mask.nonzero(as_tuple=False).squeeze(-1) + + sample_cache[sid] = dict( + clean_embeds=clean_embeds, + new_labels=new_labels, + asst_idx=asst_idx, + ) + + lengths = [int(sample_cache[int(batch["sample_id"][i])]["clean_embeds"].shape[0]) for i in range(bsz)] + Lmax = max(lengths) + mask_embed = model.get_model().embed_tokens(torch.tensor([mask_id], device=device)).squeeze(0) + + noisy_list, labels_list, p_mask_list, attn_list = [], [], [], [] + reward_list, valid_idxs = [], [] + + for i in range(bsz): + sid = int(batch["sample_id"][i]) + var_type = batch["var_type"][i] + var_param = batch["var_param"][i] + + clean_embeds = sample_cache[sid]["clean_embeds"].clone() + new_labels = sample_cache[sid]["new_labels"].clone() + asst_idx = sample_cache[sid]["asst_idx"] + + L = clean_embeds.size(0) + pmask = torch.zeros(L, dtype=torch.bool, device=device) + M = asst_idx.numel() + + if M == 0: + continue # skip badly formatted inputs + + if var_type == "TraceRL": + step_map = batch["step_map"][i] + order = torch.as_tensor(step_map, device=device) + m_len = min(M, order.numel()) + order_full = torch.full((L,), -1, dtype=torch.long, device=device) + order_full[asst_idx[:m_len]] = order[:m_len] + step_val = int(var_param) + + tgt_mask = (order_full == step_val) + pmask = tgt_mask + mask_pos = (order_full >= step_val) + + noisy_embeds = clean_embeds.clone() + noisy_embeds[mask_pos] = mask_embed + + elif var_type == "random_masking" or var_type.startswith("coupled"): + if M == 0: + # M 为 0 时直接不 mask,依靠外部 loss 里的 torch.clamp(num_mask, min=1) 避免除零,安全度过 forward + noisy_embeds = clean_embeds.clone() + pmask = torch.zeros(L, dtype=torch.bool, device=device) + else: + t = float(var_param) + # 更安全的 seed 获取方式 + mask_seed = batch.get("mask_seed", [None] * bsz)[i] + if mask_seed is None: + mask_seed = int((int(batch["sample_id"][i]) * 1000003 + int(i)) & 0x7fffffff) + + gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + rnd_cpu = torch.rand(M, generator=gen) < t + + # 如果是 coupled_comp (对比样本),需要取反 + if var_type == "coupled_comp": + rnd_cpu = ~rnd_cpu + + rnd = rnd_cpu.to(device) + + # 🌟 上帝之手:强行兜底逻辑 + # 防止全 False 导致没有 Loss 信号或引发 DDP 空 Batch 死锁 + if not rnd.any(): + # 注意:如果是 coupled_comp,由于它是 coupled_rand 的取反, + # 除非 coupled_rand 全是 True,否则 comp 不可能全 False。 + # 这里统一下一个兜底,非常安全。 + rand_pos = torch.randint(0, M, (1,), device=device) + rnd[rand_pos] = True + + pmask = torch.zeros(L, dtype=torch.bool, device=device) + pmask[asst_idx] = rnd + + noisy_embeds = clean_embeds.clone() + noisy_embeds[pmask] = mask_embed + + # elif var_type == "random_masking" or var_type.startswith("coupled"): + # t = float(var_param) + # mask_seed = batch["mask_seed"][i] + # gen = torch.Generator(device="cpu").manual_seed(int(mask_seed)) + # rnd_cpu = torch.rand(M, generator=gen) < t + + # if var_type == "coupled_comp": + # rnd_cpu = ~rnd_cpu + + # pmask = torch.zeros(L, dtype=torch.bool, device=device) + # pmask[asst_idx] = rnd_cpu.to(device) + + # noisy_embeds = clean_embeds.clone() + # noisy_embeds[pmask] = mask_embed + + if not pmask.any(): + print('************not pmask.any()**********') + continue + + noisy_list.append(left_pad_2d(noisy_embeds, Lmax, pad_val=0.0)) + labels_list.append(left_pad_1d(new_labels, Lmax, pad_val=-100)) + p_mask_list.append(left_pad_1d(pmask.to(torch.long), Lmax, pad_val=0).to(torch.bool)) + attn = torch.zeros(Lmax, dtype=torch.bool, device=device) + attn[-L:] = True + attn_list.append(attn) + reward_list.append(float(batch["reward"][i])) + valid_idxs.append(batch["idx"][i]) + + if len(noisy_list) == 0: + return None + + noisy_batch = torch.stack(noisy_list, dim=0) + labels_batch = torch.stack(labels_list, dim=0) + p_mask_batch = torch.stack(p_mask_list, dim=0) + attn_batch = torch.stack(attn_list, dim=0) + reward_batch = torch.tensor(reward_list, device=device, dtype=torch.float32) + + return noisy_batch, labels_batch, p_mask_batch, attn_batch, reward_batch, valid_idxs + + +@torch.no_grad() +def compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset, + dl, + mask_id, + config): + + model.eval() + logger.info("Computing reference log probabilities (old_lp) for all variants...") + + for batch in dl: + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + if out is None: + continue + + noisy_embeds, labels_batch, p_mask_batch, attn_batch, rewards, valid_idxs = out + + outputs = model.get_model()( + input_ids=None, + input_embeddings=noisy_embeds, + attention_mask=attn_batch, + use_cache=False, + ) + logits = outputs.logits.float() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels_batch.clone() + safe_labels[labels_batch == -100] = 0 + tok_lp = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + for i, idx in enumerate(valid_idxs): + dataset.logp_old_tok[idx] = tok_lp[i].float().cpu() + + accelerator.wait_for_everyone() + model.train() + + +def save_checkpoint(model, tokenizer, config, accelerator, name): + output_dir = Path(config.experiment.project) + output_dir.mkdir(parents=True, exist_ok=True) + + checkpoints_total_limit = config.experiment.get("checkpoints_total_limit", None) + + if accelerator.is_main_process and checkpoints_total_limit is not None: + ckpts = sorted( + [d for d in output_dir.iterdir() if d.name.startswith("checkpoint")], + key=lambda p: int(p.name.split("-")[1]), + ) + if len(ckpts) >= checkpoints_total_limit: + to_remove = ckpts[: len(ckpts) - checkpoints_total_limit + 1] + logger.info(f"removing checkpoints: {', '.join(p.name for p in to_remove)}") + for p in to_remove: + shutil.rmtree(p, ignore_errors=True) + + save_base = output_dir / "ckpt" + save_base.mkdir(exist_ok=True) + + model_to_save = accelerator.unwrap_model(model) + state_dict = accelerator.get_state_dict(model) + + if accelerator.is_main_process: + model_to_save.save_pretrained( + save_base / name, + save_function=accelerator.save, + state_dict=state_dict, + safe_serialization=True, + ) + tokenizer.save_pretrained(str(save_base / name)) + + metadata = { + "save_time": time.strftime("%Y-%m-%d %H:%M:%S"), + } + with (save_base / "metadata.json").open("w") as f: + json.dump(metadata, f, indent=2) + + logger.info(f"Saved model + tokenizer to {save_base / name}") + + +def main(): + config = get_config() + + project_name = config.experiment.project + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "./" + project_name + "/ckpt/" + config.model.optimized_name + + if config.training.enable_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.benchmark = True + torch.backends.cudnn.deterministic = False + + config.experiment.logging_dir = str(Path(config.experiment.project) / "logs") + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, + log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + set_verbosity_info() + else: + set_verbosity_error() + + if accelerator.is_main_process: + resume_wandb_run = config.wandb.resume + existing_run_id = config.wandb.get("run_id", None) + run_id = ensure_wandb_run_id(config) + if existing_run_id is None: + resume_wandb_run = False + + wandb_init_kwargs = dict( + name=config.experiment.project, + id=run_id, + resume=resume_wandb_run, + entity=config.wandb.get("entity", None), + config_exclude_keys=[], + ) + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + wandb_config.pop("experiment.resume_from_checkpoint", None) + + accelerator.init_trackers( + config.experiment.project, + config=wandb_config, + init_kwargs={"wandb": wandb_init_kwargs}, + ) + + if accelerator.is_main_process: + os.makedirs(config.experiment.project, exist_ok=True) + config_path = Path(config.experiment.project) / "config.yaml" + logging.info(f"Saving config to {config_path}") + OmegaConf.save(config, config_path) + + if config.training.seed is not None: + set_seed(config.training.seed) + + logger.info("Loading LaViDa model and tokenizer from official LaViDa codebase") + + tokenizer, model, image_processor, _ = load_pretrained_model( + pretrained_model, None, LAVIDA_MODEL_NAME, attn_implementation="sdpa", device_map=accelerator.device + ) + model = model.to(accelerator.device) + + special_tokens = {"additional_special_tokens": [DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN]} + num_new = tokenizer.add_special_tokens(special_tokens) + if num_new > 0: + model.resize_token_embeddings(len(tokenizer)) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + if getattr(config.training, "gradient_checkpointing_enable", False): + try: + model.enable_input_require_grads() + except Exception: + pass + try: + model.gradient_checkpointing_enable() + except Exception: + pass + + mask_id = 126336 + + optimizer_config = config.optimizer.params + no_decay = ["bias", "layer_norm.weight", "mlm_ln.weight", "embeddings.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and not any(nd in n for nd in no_decay)], + "weight_decay": optimizer_config.weight_decay, + }, + { + "params": [p for n, p in model.named_parameters() if + p.requires_grad and any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + ] + + optimizer_type = config.optimizer.name + if optimizer_type == "adamw": + optimizer = AdamW( + optimizer_grouped_parameters, + lr=optimizer_config.learning_rate, + betas=(optimizer_config.beta1, optimizer_config.beta2), + weight_decay=optimizer_config.weight_decay, + eps=optimizer_config.epsilon, + ) + else: + raise ValueError(f"Optimizer {optimizer_type} not supported") + + + logger.info("Preprocessing Data") + + with open("./" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", 'r') as f: + dataset_load = json.load(f) + + dataset_lm = RLVariantDatasetLLADAV(dataset_load, config) + + logger.info("Preparing model, optimizer and dataloaders") + + total_batch_size_lm = config.training.batch_size_lm * accelerator.num_processes * config.training.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(dataset_lm) / total_batch_size_lm) + num_train_epochs = config.training.num_train_epochs + max_train_steps = num_update_steps_per_epoch * num_train_epochs + 1 + + lr_scheduler = get_scheduler( + config.lr_scheduler.scheduler, + optimizer=optimizer, + num_training_steps=max_train_steps, + num_warmup_steps=config.lr_scheduler.params.warmup_steps, + min_lr_scale=config.lr_scheduler.params.min_lr_scale + ) + + train_dataloader_lm = DataLoader( + dataset_lm, + batch_size=config.training.batch_size_lm, + sampler=None, + collate_fn=collate_variants, + num_workers=0, + shuffle=False + ) + + model, optimizer, lr_scheduler, train_dataloader_lm = accelerator.prepare( + model, optimizer, lr_scheduler, train_dataloader_lm + ) + + logger.info("***** Running inference *****") + compute_logp_old_tok_parallel( + accelerator, + model, + tokenizer, + image_processor, + dataset_lm, + train_dataloader_lm, + mask_id=mask_id, + config=config + ) + + logger.info("***** Running training *****") + logger.info(f" Num variants = {len(dataset_lm)}") + logger.info(f" Num training steps = {max_train_steps}") + logger.info(f" Instantaneous batch size per device = {config.training.batch_size_lm}") + logger.info(f" Total train batch size (parallel * accum) = {total_batch_size_lm}") + logger.info(f" Gradient Accumulation steps = {config.training.gradient_accumulation_steps}") + + first_epoch = 0 + data_time_m = AverageMeter() + end = time.time() + grad_accum = config.training.gradient_accumulation_steps + + def forward_process(noisy_embeds, labels, p_mask_lm, attn_mask, adv, logp_old_tok): + noisy_embeds.requires_grad_(True) + outputs = model.get_model()( + input_ids=None, + input_embeddings=noisy_embeds, + attention_mask=attn_mask, + use_cache=False, + ) + logits = outputs.logits.float() + + B, T, V = logits.shape + adv = torch.as_tensor(adv, device=noisy_embeds.device).detach() + + log_probs = F.log_softmax(logits, dim=-1) + safe_labels = labels.clone() + safe_labels[labels == -100] = 0 + logp_new_tok = log_probs.gather(dim=-1, index=safe_labels.unsqueeze(-1)).squeeze(-1) + + ratio = torch.exp(logp_new_tok - logp_old_tok) + clipped = torch.clamp(ratio, 1 - config.training.eps, 1 + config.training.eps) + adv_tok = adv.unsqueeze(1) + + surrogate_tok = torch.min(ratio * adv_tok, clipped * adv_tok) + surrogate_tok = surrogate_tok * p_mask_lm + num_mask = torch.clamp(p_mask_lm.sum(dim=1), min=1) + surrogate_tok = surrogate_tok.sum(dim=1) / num_mask + policy_loss = - (surrogate_tok.sum() / B) + + # KL penalty + kl_loss = torch.tensor(0.0, device=policy_loss.device) + if config.training.beta > 0: + kl_seq = logp_new_tok - logp_old_tok + if config.training.use_kl_estimator_k3: + kl_seq = (-kl_seq).exp() - 1.0 + kl_seq + kl_seq = (kl_seq * p_mask_lm).sum(dim=1) + kl_loss = config.training.beta * kl_seq.sum() / B + total_loss = policy_loss + kl_loss + else: + total_loss = policy_loss + + return total_loss + + from tqdm.auto import tqdm + + cprint("start rl training...", "red") + for epoch in range(first_epoch, num_train_epochs): + model.train() + progress_bar = tqdm( + train_dataloader_lm, + desc=f"Epoch {epoch+1}/{num_train_epochs}", + disable=not accelerator.is_local_main_process, + dynamic_ncols=True, + leave=True + ) + + step = 0 + for step, batch in enumerate(progress_bar, start=1): + data_time_m.update(time.time() - end) + + out = prepare_batch_noisy_embeds_rl( + accelerator=accelerator, + model=model, + tokenizer=tokenizer, + image_processor=image_processor, + batch=batch, + mask_id=mask_id, + dataset_media_root=config.dataset.get("image_root", None), + ) + + if out is None: + continue + + noisy_embeds, labels, p_mask_lm, attn_mask, rewards, valid_idxs = out + + # Align old log probabilities shape with current noisy_embeds + L_current = noisy_embeds.shape[1] + old_lp_batch = torch.zeros((len(valid_idxs), L_current), device=accelerator.device) + for i, idx in enumerate(valid_idxs): + old_t = dataset_lm.logp_old_tok[idx] + l_min = min(L_current, old_t.shape[0]) + old_lp_batch[i, :l_min] = old_t[:l_min].to(accelerator.device) + + loss_lm = forward_process( + noisy_embeds=noisy_embeds, + labels=labels, + p_mask_lm=p_mask_lm, + attn_mask=attn_mask, + adv=rewards, + logp_old_tok=old_lp_batch + ) + + loss_lm = loss_lm / grad_accum + accelerator.backward(loss_lm) + + if step % grad_accum == 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + # Catch remaining gradients + if step % grad_accum != 0: + current_lr = optimizer.param_groups[0]['lr'] + # current_global_step = global_step_offset + (step // grad_accum) + + accelerator.log({ + "train/loss": loss_lm.item() * grad_accum, + "train/learning_rate": current_lr, + # "train/iteration": config.experiment.current_epoch, + }, step=(step // grad_accum)) + + if config.training.max_grad_norm is not None: + accelerator.clip_grad_norm_(model.parameters(), config.training.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=True) + torch.cuda.empty_cache() + + accelerator.wait_for_everyone() + + save_checkpoint(model, tokenizer, config, accelerator, config.model.optimized_name) + if config.experiment.current_epoch % config.experiment.save_every == 0: + save_checkpoint(model, tokenizer, config, accelerator, f"epoch-{config.experiment.current_epoch}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/wandb/run-20260602_132409-w3wdpl8c/files/config.yaml b/wandb/run-20260602_132409-w3wdpl8c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..877efa9058777b403f9c9bcde20b51b234f3e04b --- /dev/null +++ b/wandb/run-20260602_132409-w3wdpl8c/files/config.yaml @@ -0,0 +1,288 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/train/rl_lavida.py + e: + 39min72dan3ke69rya7wp8v75kpnhobz: + args: + - config=configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml + - wandb.run_id=w3wdpl8c + - experiment.current_epoch=1 + codePath: train/rl_lavida.py + codePathLocal: train/rl_lavida.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "626439921664" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python3.10 + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/train/rl_lavida.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL + startedAt: "2026-06-02T05:24:09.345401Z" + writerId: 39min72dan3ke69rya7wp8v75kpnhobz + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 53 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 98 + "3": + - 2 + - 5 + - 13 + - 14 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.52.4 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_sft_train_cleaned_10k +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: 32 +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 1 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_revvideo_orig10k_4gpu_len128/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_revvideo_orig10k_4gpu_len128 +experiment.save_every: + value: 5 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 10 +lr_scheduler.params.learning_rate: + value: 5e-07 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/hf/lavida-llada-v1.0-instruct +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 5e-07 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.05 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: w3wdpl8c diff --git a/wandb/run-20260602_132409-w3wdpl8c/files/diff.patch b/wandb/run-20260602_132409-w3wdpl8c/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..66f5a5d79b3fed7e41943c16b223f99b5237007d --- /dev/null +++ b/wandb/run-20260602_132409-w3wdpl8c/files/diff.patch @@ -0,0 +1,376 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..176e35b 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -97,9 +97,9 @@ class Conversation: + elif self.sep_style == SeparatorStyle.LLAMA_3: + if self.tokenizer is None: + if self.version == "llama_v3": +- self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/hf/Meta-Llama-3-8B-Instruct', trust_remote_code=True, local_files_only=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +@@ -384,7 +384,7 @@ conv_llava_llama_2 = Conversation( + + def safe_load_tokenizer(tokenizer_id): + try: +- return AutoTokenizer.from_pretrained(tokenizer_id) ++ return AutoTokenizer.from_pretrained(tokenizer_id, local_files_only=True) + except Exception: + return None + +@@ -397,7 +397,7 @@ conv_llava_llama_3 = Conversation( + sep="<|eot_id|>", + sep_style=SeparatorStyle.LLAMA_3, + tokenizer_id="meta-llama/Meta-Llama-3-8B-Instruct", +- tokenizer=safe_load_tokenizer("/data_storage/lbw/shared/pretrained_models/Meta-Llama-3-8B-Instruct"), ++ tokenizer=safe_load_tokenizer("/data4/runqing/hf/Meta-Llama-3-8B-Instruct"), + stop_token_ids=[128009], + ) + +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260602_132409-w3wdpl8c/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch b/wandb/run-20260602_132409-w3wdpl8c/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch new file mode 100644 index 0000000000000000000000000000000000000000..66f5a5d79b3fed7e41943c16b223f99b5237007d --- /dev/null +++ b/wandb/run-20260602_132409-w3wdpl8c/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch @@ -0,0 +1,376 @@ +diff --git a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +index 0ca6138..4c35ee9 100644 +--- a/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml ++++ b/accelerate_configs/1_node_8_gpus_deepspeed_zero2.yaml +@@ -10,7 +10,7 @@ downcast_bf16: 'no' + main_training_function: main + mixed_precision: bf16 + num_machines: 1 +-num_processes: 8 ++num_processes: 4 + rdzv_backend: static + same_network: true + tpu_env: [] +diff --git a/configs/lladav_eval.yaml b/configs/lladav_eval.yaml +index 3e07fbd..337824b 100644 +--- a/configs/lladav_eval.yaml ++++ b/configs/lladav_eval.yaml +@@ -7,17 +7,17 @@ system: + HTTP_PROXY: null + HF_HOME: null + env_name: "" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ envs_dir: null ++ base_dir: "/data4/runqing/dLLM-RL" + +-model: "/abs/path/of/model" ++model: "/data4/runqing/dLLM-RL/LLaDA-V" + model_base: "lladav" + answer_must_in_box: True + + dataset: + eval_dataset: "SEEDBench_IMG_32" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + execute: + num_chunk: 128 +@@ -35,4 +35,3 @@ rollout: + further_horizon: 128 + output_unmasking_history: True + use_cache: True +- +diff --git a/configs/rl_lladav.yaml b/configs/rl_lladav.yaml +index 7e38050..a07f620 100644 +--- a/configs/rl_lladav.yaml ++++ b/configs/rl_lladav.yaml +@@ -5,9 +5,9 @@ wandb: + system: + HTTP_PROXY: null + HF_HOME: null +- env_name: "env_name" +- envs_dir: "/abs/path/of/envs" +- base_dir: "/abs/path/of/dLLM-RL" ++ env_name: "/data8/runqing/miniconda/envs/dllm-rl" ++ envs_dir: "/data8/runqing/miniconda/envs/dllm-rl" ++ base_dir: "/data4/runqing/dLLM-RL" + + experiment: + project: "rl_lladav" +@@ -22,17 +22,17 @@ experiment: + node_index: 0 + + model: +- pretrained_model: "/abs/path/of/sft-ckpt" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + model_base: "lladav" +- vq_model_path: "/abs/path/of/magvitv2" ++ vq_model_path: "/data4/runqing/hf/magvitv2" + image_resolution: 512 + + dataset: + train_dataset: "SEEDBench_IMG_32" + optimization_data: "rl_data" + data_type: "mmu" +- image_root: "/abs/path/of/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + rollout: + num_task_per_step: 128 +diff --git a/configs/sft_lladav.yaml b/configs/sft_lladav.yaml +index 6557210..001a6e2 100644 +--- a/configs/sft_lladav.yaml ++++ b/configs/sft_lladav.yaml +@@ -7,13 +7,13 @@ experiment: + num_node: 1 + + model: +- pretrained_model: "/abs/path/of/model" ++ pretrained_model: "/data4/runqing/dLLM-RL/LLaDA-V" + optimized_name: "optimized" + + dataset: +- optimization_data: "sft_test" # ./data/sft_test.json ++ optimization_data: "sft_test_v" # ./data/sft_test.json + data_type: "mmu" +- image_root: "/base_dir/data/images" ++ image_root: "/data4/runqing/dLLM-RL/data/images" + + training: + gradient_checkpointing_enable: False +diff --git a/data/preprocess_sft.ipynb b/data/preprocess_sft.ipynb +index e0b183f..9f94fa7 100644 +--- a/data/preprocess_sft.ipynb ++++ b/data/preprocess_sft.ipynb +@@ -30,15 +30,15 @@ + "#eos_token = \"<|im_end|>\"\n", + "\n", + "# llada and mmada:\n", +- "#system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", +- "#eos_token = \"<|eot_id|>\"\n", ++ "system_prompts = \"\"\"<|startoftext|><|start_header_id|>user<|end_header_id|>You need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|eot_id|><|startoftext|><|start_header_id|>assistant<|end_header_id|>\\n\"\"\"\n", ++ "eos_token = \"<|eot_id|>\"\n", + "\n", + "# trado and sdar:\n", + "# non-cot prompt, we used this as demon example to compare sft methods\n", +- "system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", ++ "# system_prompts = '''<|im_start|>user\\nYou need to put your final answer in \\\\boxed{}. This is the problem:\\n{{problem}}<|im_end|>\\n<|im_start|>assistant\\n'''\n", + "# cot prompt, trado and sdar are not cot-default, needs cot prompt to activate reasoning ability.\n", + "#system_prompts = '''<|im_start|>user\\n{{problem}}\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n<|im_start|>assistant\\n'''\n", +- "eos_token = \"<|im_end|>\"\n", ++ "# eos_token = \"<|im_end|>\"\n", + "\n", + "\n", + "def get_prompt(data_i):\n", +@@ -67,6 +67,87 @@ + "with open(\"./sft_openr1math_sdar.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(processed_data, f, indent=2, ensure_ascii=False)" + ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": 2, ++ "id": "32bbd872", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "100\n" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reverse_sft_data_shuffle_option_3k.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reverse_rl_data_shuffle_option_100.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ " for data in raw_data:\n", ++ " if save_num < 100:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] ++ }, ++ { ++ "cell_type": "code", ++ "execution_count": null, ++ "id": "fdfde30f", ++ "metadata": {}, ++ "outputs": [ ++ { ++ "name": "stdout", ++ "output_type": "stream", ++ "text": [ ++ "200\n" ++ ] ++ }, ++ { ++ "ename": "", ++ "evalue": "", ++ "output_type": "error", ++ "traceback": [ ++ "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n", ++ "\u001b[1;31m请查看单元格中的代码,以确定故障的可能原因。\n", ++ "\u001b[1;31m单击此处了解详细信息。\n", ++ "\u001b[1;31m有关更多详细信息,请查看 Jupyter log。" ++ ] ++ } ++ ], ++ "source": [ ++ "import json\n", ++ "import random\n", ++ "random.seed(2026)\n", ++ "\n", ++ "\n", ++ "data_path = '/data4/runqing/dLLM-RL/data/video_reversal_sft_test.json'\n", ++ "save_path = '/data4/runqing/dLLM-RL/data/video_reversal_rl_eval.json'\n", ++ "save_data_list = []\n", ++ "save_num = 0\n", ++ "with open(data_path, 'r') as fin, \\\n", ++ " open(save_path, 'w') as fout:\n", ++ " raw_data = json.load(fin)\n", ++ "\n", ++ " # random.shuffle(raw_data)\n", ++ " for did, data in enumerate(raw_data):\n", ++ " # if did > 1999:\n", ++ " if save_num < 200:\n", ++ " save_data_list.append(data)\n", ++ " save_num += 1\n", ++ " json.dump(save_data_list, fout, indent=2, ensure_ascii=False)\n", ++ " print(len(save_data_list))\n" ++ ] + } + ], + "metadata": { +@@ -85,7 +166,7 @@ + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +- "version": "3.10.18" ++ "version": "3.10.19" + } + }, + "nbformat": 4, +diff --git a/rl_v.py b/rl_v.py +index 8f2c0f1..79d2e5b 100644 +--- a/rl_v.py ++++ b/rl_v.py +@@ -15,7 +15,9 @@ if __name__ == "__main__": + def env_prefix(env) -> str: + return ( + "source ~/.bashrc && " +- f"source activate {env} && " ++ "source /data8/runqing/miniconda/bin/activate && " ++ "conda init && " ++ f"conda activate {env} && " + ) + + config = get_config() +diff --git a/sample/llava/conversation.py b/sample/llava/conversation.py +index 4068fc8..176e35b 100644 +--- a/sample/llava/conversation.py ++++ b/sample/llava/conversation.py +@@ -97,9 +97,9 @@ class Conversation: + elif self.sep_style == SeparatorStyle.LLAMA_3: + if self.tokenizer is None: + if self.version == "llama_v3": +- self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/hf/Meta-Llama-3-8B-Instruct', trust_remote_code=True, local_files_only=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +@@ -384,7 +384,7 @@ conv_llava_llama_2 = Conversation( + + def safe_load_tokenizer(tokenizer_id): + try: +- return AutoTokenizer.from_pretrained(tokenizer_id) ++ return AutoTokenizer.from_pretrained(tokenizer_id, local_files_only=True) + except Exception: + return None + +@@ -397,7 +397,7 @@ conv_llava_llama_3 = Conversation( + sep="<|eot_id|>", + sep_style=SeparatorStyle.LLAMA_3, + tokenizer_id="meta-llama/Meta-Llama-3-8B-Instruct", +- tokenizer=safe_load_tokenizer("/data_storage/lbw/shared/pretrained_models/Meta-Llama-3-8B-Instruct"), ++ tokenizer=safe_load_tokenizer("/data4/runqing/hf/Meta-Llama-3-8B-Instruct"), + stop_token_ids=[128009], + ) + +diff --git a/sample/llava/model/builder.py b/sample/llava/model/builder.py +index 3db5302..96e33b5 100644 +--- a/sample/llava/model/builder.py ++++ b/sample/llava/model/builder.py +@@ -234,7 +234,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) ++ ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/conversation.py b/train/llava/conversation.py +index 4068fc8..d284a23 100644 +--- a/train/llava/conversation.py ++++ b/train/llava/conversation.py +@@ -99,7 +99,7 @@ class Conversation: + if self.version == "llama_v3": + self.tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct', trust_remote_code=True) + elif self.version == "llava_llada": +- self.tokenizer = AutoTokenizer.from_pretrained('/data_storage/lbw/shared/pretrained_models/LLaDA-V', trust_remote_code=True) ++ self.tokenizer = AutoTokenizer.from_pretrained('/data4/runqing/dLLM-RL/LLaDA-V', trust_remote_code=True) + else: + raise ValueError("The tokenizer is not available. Make sure you have the necessary permissions.") + chat_template_messages = [{"role": "system", "content": self.system}] +diff --git a/train/llava/model/builder.py b/train/llava/model/builder.py +index 3db5302..aafab95 100644 +--- a/train/llava/model/builder.py ++++ b/train/llava/model/builder.py +@@ -233,8 +233,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l + model = LlavaGemmaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, config=cfg_pretrained, attn_implementation=attn_implementation, **kwargs) + elif "llada" in model_name.lower(): + from llava.model.language_model.llava_llada import LlavaLLaDAConfig ++ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + +- tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) ++ # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + if customized_config is None: + llada_cfg = LlavaLLaDAConfig.from_pretrained(model_path) + else: +diff --git a/train/llava/model/language_model/llava_llada.py b/train/llava/model/language_model/llava_llada.py +index 3da28dc..8b17da3 100644 +--- a/train/llava/model/language_model/llava_llada.py ++++ b/train/llava/model/language_model/llava_llada.py +@@ -84,7 +84,7 @@ class LlavaLLaDAModelLM(LLaDAModelLM, LlavaMetaForCausalLM): + dpo_forward: Optional[bool] = None, + cache_position=None, + ) -> Union[Tuple, CausalLMOutputWithPast]: +- ++ conversation_ids = None + if inputs_embeds is None and attention_mask is not None: + # donate multi-dialogue + (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, conversation_ids) = self.prepare_inputs_labels_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, labels, images, modalities, image_sizes, is_llada=True) +diff --git a/train/rl_llada.py b/train/rl_llada.py +index 916ce23..00693be 100644 +--- a/train/rl_llada.py ++++ b/train/rl_llada.py +@@ -96,7 +96,7 @@ def main(): + accelerator = Accelerator( + gradient_accumulation_steps=config.training.gradient_accumulation_steps, + mixed_precision=config.training.mixed_precision, +- log_with=None, ++ log_with='wandb', + project_dir=config.experiment.logging_dir, + split_batches=True, + ) +diff --git a/train/utils.py b/train/utils.py +index bf9ad65..31bf60f 100644 +--- a/train/utils.py ++++ b/train/utils.py +@@ -4,6 +4,7 @@ import torch + import torch.nn.functional as F + from omegaconf import DictConfig, ListConfig, OmegaConf + from typing import Any, List, Tuple, Union ++import wandb + + + ################################################## +@@ -48,6 +49,14 @@ def flatten_omega_conf(cfg: Any, resolve: bool = False) -> List[Tuple[str, Any]] + return ret + + ++def ensure_wandb_run_id(cfg: Any) -> str: ++ run_id = OmegaConf.select(cfg, "wandb.run_id", default=None) ++ if run_id is None: ++ run_id = wandb.util.generate_id() ++ cfg.wandb.run_id = run_id ++ return run_id ++ ++ + ################################################## + # training utils + ################################################## +@@ -210,4 +219,4 @@ def image_transform_squash(image, resolution=256, normalize=True): + image = transforms.ToTensor()(image) + if normalize: + image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True)(image) +- return image +\ No newline at end of file ++ return image diff --git a/wandb/run-20260602_132409-w3wdpl8c/files/output.log b/wandb/run-20260602_132409-w3wdpl8c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..472fbb8187f1bc6d5af3fcb0fa994e7dd99c2859 --- /dev/null +++ b/wandb/run-20260602_132409-w3wdpl8c/files/output.log @@ -0,0 +1,77 @@ +06/02/2026 13:24:10 - INFO - root - Saving config to rl_lavida_revvideo_orig10k_4gpu_len128/config.yaml +06/02/2026 13:24:10 - INFO - __main__ - Loading LaViDa model and tokenizer from official LaViDa codebase +Rank 0: Loaded LLaVA model: /data4/runqing/hf/lavida-llada-v1.0-instruct +06/02/2026 13:24:10 - INFO - llava.model.language_model.llada.modeling_llada - Initializing model parameters... +Rank 0: Loading vision tower: google/siglip-so400m-patch14-384 +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: d1ac3cda-1fc7-44f2-be73-c0356e9ef7ea)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:24:20 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: d1ac3cda-1fc7-44f2-be73-c0356e9ef7ea)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +Retrying in 1s [Retry 1/5]. +06/02/2026 13:24:20 - WARNING - huggingface_hub.utils._http - Retrying in 1s [Retry 1/5]. +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: ca55e12b-dcd0-4f59-b469-1fc6d2ba206b)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:24:31 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: ca55e12b-dcd0-4f59-b469-1fc6d2ba206b)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +Retrying in 2s [Retry 2/5]. +06/02/2026 13:24:31 - WARNING - huggingface_hub.utils._http - Retrying in 2s [Retry 2/5]. +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: b2e303b4-dbc0-4a93-9b42-8786964d0ef8)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:24:43 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: b2e303b4-dbc0-4a93-9b42-8786964d0ef8)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +Retrying in 4s [Retry 3/5]. +06/02/2026 13:24:43 - WARNING - huggingface_hub.utils._http - Retrying in 4s [Retry 3/5]. +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: b9877f7f-40c8-4019-bb88-8c8e7ea56b90)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:24:57 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: b9877f7f-40c8-4019-bb88-8c8e7ea56b90)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +Retrying in 8s [Retry 4/5]. +06/02/2026 13:24:57 - WARNING - huggingface_hub.utils._http - Retrying in 8s [Retry 4/5]. +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: 2e6f7492-6364-4acd-8f01-0f318b6c2b2e)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:25:15 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: 2e6f7492-6364-4acd-8f01-0f318b6c2b2e)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +Retrying in 8s [Retry 5/5]. +06/02/2026 13:25:15 - WARNING - huggingface_hub.utils._http - Retrying in 8s [Retry 5/5]. +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: a269f122-292e-4057-97f6-fca19c679376)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:25:34 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: a269f122-292e-4057-97f6-fca19c679376)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: f9e14588-174a-4b5d-af8b-2676931e3703)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:25:44 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: f9e14588-174a-4b5d-af8b-2676931e3703)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +Retrying in 1s [Retry 1/5]. +06/02/2026 13:25:44 - WARNING - huggingface_hub.utils._http - Retrying in 1s [Retry 1/5]. +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: c7c7f74d-efd1-47cf-9fcf-9f2cb5629d75)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:25:55 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: c7c7f74d-efd1-47cf-9fcf-9f2cb5629d75)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +Retrying in 2s [Retry 2/5]. +06/02/2026 13:25:55 - WARNING - huggingface_hub.utils._http - Retrying in 2s [Retry 2/5]. +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: 9317c698-eb18-49fc-b00d-6955a0c69b96)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:26:07 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: 9317c698-eb18-49fc-b00d-6955a0c69b96)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +Retrying in 4s [Retry 3/5]. +06/02/2026 13:26:07 - WARNING - huggingface_hub.utils._http - Retrying in 4s [Retry 3/5]. +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: d22818e3-876c-4d1a-91e4-4fc89fd05980)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:26:21 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: d22818e3-876c-4d1a-91e4-4fc89fd05980)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +Retrying in 8s [Retry 4/5]. +06/02/2026 13:26:21 - WARNING - huggingface_hub.utils._http - Retrying in 8s [Retry 4/5]. +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: 02b39369-61e8-49ef-926b-08ffcb0866eb)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:26:39 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: 02b39369-61e8-49ef-926b-08ffcb0866eb)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +Retrying in 8s [Retry 5/5]. +06/02/2026 13:26:39 - WARNING - huggingface_hub.utils._http - Retrying in 8s [Retry 5/5]. +'(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: ff4da4a1-79b2-4ae1-b851-2d2fecc34afa)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +06/02/2026 13:26:57 - WARNING - huggingface_hub.utils._http - '(MaxRetryError('HTTPSConnectionPool(host=\'huggingface.co\', port=443): Max retries exceeded with url: /google/siglip-so400m-patch14-384/resolve/main/config.json (Caused by NewConnectionError("HTTPSConnection(host=\'huggingface.co\', port=443): Failed to establish a new connection: [Errno 101] Network is unreachable"))'), '(Request ID: ff4da4a1-79b2-4ae1-b851-2d2fecc34afa)')' thrown while requesting HEAD https://huggingface.co/google/siglip-so400m-patch14-384/resolve/main/config.json +Loading checkpoint shards: 100%|██████████| 4/4 [00:03<00:00, 1.06it/s] +Rank 0: Model Class: LlavaLladaForMaskedDiffusion +06/02/2026 13:27:01 - INFO - __main__ - Preprocessing Data +06/02/2026 13:27:01 - INFO - __main__ - Preparing model, optimizer and dataloaders +/data8/runqing/miniconda/envs/dllm-rl/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST']. + warnings.warn( +Before initializing optimizer states +MA 16.71 GB Max_MA 16.71 GB CA 21.04 GB Max_CA 21 GB +CPU Virtual Memory: used = 103.09 GB, percent = 5.1% +After initializing optimizer states +MA 16.71 GB Max_MA 16.71 GB CA 21.04 GB Max_CA 21 GB +CPU Virtual Memory: used = 115.98 GB, percent = 5.8% +After initializing ZeRO optimizer +MA 16.71 GB Max_MA 16.71 GB CA 21.04 GB Max_CA 21 GB +CPU Virtual Memory: used = 116.97 GB, percent = 5.8% +06/02/2026 13:27:49 - INFO - __main__ - ***** Running inference ***** +06/02/2026 13:27:49 - INFO - __main__ - Computing reference log probabilities (old_lp) for all variants... +06/02/2026 13:28:14 - INFO - __main__ - ***** Running training ***** +06/02/2026 13:28:14 - INFO - __main__ - Num variants = 16 +06/02/2026 13:28:14 - INFO - __main__ - Num training steps = 3 +06/02/2026 13:28:14 - INFO - __main__ - Instantaneous batch size per device = 1 +06/02/2026 13:28:14 - INFO - __main__ - Total train batch size (parallel * accum) = 8 +06/02/2026 13:28:14 - INFO - __main__ - Gradient Accumulation steps = 2 +start rl training... +Epoch 1/1: 50%|█████ | 2/4 [00:55<00:53, 26.69s/it]wandb: WARNING Tried to log to step 1 that is less than the current step 2. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +Epoch 1/1: 100%|██████████| 4/4 [01:28<00:00, 22.03s/it] +06/02/2026 13:30:36 - INFO - __main__ - Saved model + tokenizer to rl_lavida_revvideo_orig10k_4gpu_len128/ckpt/optimized diff --git a/wandb/run-20260602_132409-w3wdpl8c/files/requirements.txt b/wandb/run-20260602_132409-w3wdpl8c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/wandb/run-20260602_132409-w3wdpl8c/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/wandb/run-20260602_132409-w3wdpl8c/files/wandb-metadata.json b/wandb/run-20260602_132409-w3wdpl8c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f51d5db8ff99038dfb976641898f50609167ab63 --- /dev/null +++ b/wandb/run-20260602_132409-w3wdpl8c/files/wandb-metadata.json @@ -0,0 +1,94 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-02T05:24:09.345401Z", + "args": [ + "config=configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml", + "wandb.run_id=w3wdpl8c", + "experiment.current_epoch=1" + ], + "program": "/data4/runqing/dLLM-RL/train/rl_lavida.py", + "codePath": "train/rl_lavida.py", + "codePathLocal": "train/rl_lavida.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python3.10", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "626439921664" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "39min72dan3ke69rya7wp8v75kpnhobz" +} \ No newline at end of file diff --git a/wandb/run-20260602_132409-w3wdpl8c/files/wandb-summary.json b/wandb/run-20260602_132409-w3wdpl8c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a63773ccf3fc9705809160d16ecb22d47a672641 --- /dev/null +++ b/wandb/run-20260602_132409-w3wdpl8c/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":386.880743673,"_timestamp":1.7803781824325833e+09,"_wandb":{"runtime":386},"_step":2,"reward/acc":0.02734375,"reward/unbiased_bonus":0,"reward/t_grpo_bonus":0.29531250000000037,"reward/avg_response_length":35.7275390625,"reward/train_step":1,"train/loss":-0.5786913633346558,"train/learning_rate":1.249999999999999e-07} \ No newline at end of file diff --git a/wandb/run-20260602_132409-w3wdpl8c/logs/debug-core.log b/wandb/run-20260602_132409-w3wdpl8c/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..48d464d2a70507abb8104bdb57bc21e81f6984d2 --- /dev/null +++ b/wandb/run-20260602_132409-w3wdpl8c/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-06-02T13:24:09.413473095+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpg2y1yzy6/port-750246.txt","pid":750246,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-02T13:24:09.414118386+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":750246} +{"time":"2026-06-02T13:24:09.414104074+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-750246-750702-1581898729/socket","Net":"unix"}} +{"time":"2026-06-02T13:24:09.594259446+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-02T13:24:09.601372911+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T13:24:10.210110924+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T13:24:15.544183983+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"mn6vlfy7llbl"} +{"time":"2026-06-02T13:30:37.297883633+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"mn6vlfy7llbl"} +{"time":"2026-06-02T13:30:37.497330299+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"mn6vlfy7llbl"} +{"time":"2026-06-02T13:30:37.497771152+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T13:30:37.498539849+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T13:30:37.889214924+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-02T13:30:37.889283337+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-02T13:30:37.889306727+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-02T13:30:37.889311217+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-02T13:30:37.889424945+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-750246-750702-1581898729/socket","Net":"unix"}} +{"time":"2026-06-02T13:30:37.88946656+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-02T13:30:37.889493105+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-02T13:30:37.889517599+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260602_132409-w3wdpl8c/logs/debug-internal.log b/wandb/run-20260602_132409-w3wdpl8c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..111216a608e5c3440cfb14630ea0ff98978e8ded --- /dev/null +++ b/wandb/run-20260602_132409-w3wdpl8c/logs/debug-internal.log @@ -0,0 +1,31 @@ +{"time":"2026-06-02T13:24:09.601583498+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-02T13:24:10.209180228+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-02T13:24:10.209353459+08:00","level":"INFO","msg":"stream: created new stream","id":"w3wdpl8c"} +{"time":"2026-06-02T13:24:10.209455145+08:00","level":"INFO","msg":"handler: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T13:24:10.210058012+08:00","level":"INFO","msg":"stream: started","id":"w3wdpl8c"} +{"time":"2026-06-02T13:24:10.211066978+08:00","level":"INFO","msg":"writer: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T13:24:10.211115685+08:00","level":"INFO","msg":"sender: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T13:24:10.214138416+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-02T13:24:15.485183024+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052415Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9beb4771673e63a1369ebf8e60ea8e0f0361fbcfe526561def2691f9b30953d7"} +{"time":"2026-06-02T13:24:15.48535748+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt18B52C3A083B43A729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260602_132409-w3wdpl8c/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052415Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9beb4771673e63a1369ebf8e60ea8e0f0361fbcfe526561def2691f9b30953d7, Size: 2982}"} +{"time":"2026-06-02T13:24:15.48541615+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052415Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=71dff7cd731896baf2d70c68f0ddb6434ca6e7e8878ae31b2c50733012b82bb0"} +{"time":"2026-06-02T13:24:15.485572969+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json18B52C3A0966F8E14b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260602_132409-w3wdpl8c/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052415Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=71dff7cd731896baf2d70c68f0ddb6434ca6e7e8878ae31b2c50733012b82bb0, Size: 2678}"} +{"time":"2026-06-02T13:24:15.493189971+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/train/rl_lavida.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052415Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=62754ec699c18dee1b336da5e7ab4782f6c8bd8dc653da2d4eaaff0d686b2511"} +{"time":"2026-06-02T13:24:15.4933578+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/train/rl_lavida.pynetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/train/rl_lavida.py18B52C3A08F3DB3017e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260602_132409-w3wdpl8c/files/code/train/rl_lavida.py, Name: code/train/rl_lavida.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/train/rl_lavida.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052415Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=62754ec699c18dee1b336da5e7ab4782f6c8bd8dc653da2d4eaaff0d686b2511, Size: 32927}"} +{"time":"2026-06-02T13:24:15.49364585+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052415Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c4b6af729339816da354896b613ad42341b7e69e0de0c242995ec595fc65a6cf"} +{"time":"2026-06-02T13:24:15.493738545+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/diff.patchnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/diff.patch18B52C3A090FE60217e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260602_132409-w3wdpl8c/files/diff.patch, Name: diff.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/diff.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052415Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c4b6af729339816da354896b613ad42341b7e69e0de0c242995ec595fc65a6cf, Size: 15262}"} +{"time":"2026-06-02T13:24:15.495441235+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052415Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2bd74dbaaf68401e470b2ffd1169bc57fe76365948b53e054ff1ff78486bceb6"} +{"time":"2026-06-02T13:24:15.495626418+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patchnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch18B52C3A09074BBD17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260602_132409-w3wdpl8c/files/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Name: diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/diff_10b4fd1cba83e4496c20585dc2fd01dc02f20255.patch?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052415Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2bd74dbaaf68401e470b2ffd1169bc57fe76365948b53e054ff1ff78486bceb6, Size: 15262}"} +{"time":"2026-06-02T13:29:09.910267134+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":1,"current":2} +{"time":"2026-06-02T13:30:37.412583187+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T053037Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f686fa933dabe564cd862e09e153359318aa5e44e40a123c9f85eb07af1f513a"} +{"time":"2026-06-02T13:30:37.412708998+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json18B52C92F605AD3C4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260602_132409-w3wdpl8c/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T053037Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f686fa933dabe564cd862e09e153359318aa5e44e40a123c9f85eb07af1f513a, Size: 331}"} +{"time":"2026-06-02T13:30:37.414662303+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T053037Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0b990da3645537b9704acb7c4542fa16f4551014cbd3854d9bd759a5a4a238fa"} +{"time":"2026-06-02T13:30:37.41484371+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml18B52C92F5B0757B17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260602_132409-w3wdpl8c/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T053037Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0b990da3645537b9704acb7c4542fa16f4551014cbd3854d9bd759a5a4a238fa, Size: 8012}"} +{"time":"2026-06-02T13:30:37.421994158+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T053037Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=596e4ae599496d709f9efab8a42d703603c4f7c7b60934218431316da79735da"} +{"time":"2026-06-02T13:30:37.422060307+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log18B52C92F4EBFFDC29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/wandb/run-20260602_132409-w3wdpl8c/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T053037Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=596e4ae599496d709f9efab8a42d703603c4f7c7b60934218431316da79735da, Size: 16330}"} +{"time":"2026-06-02T13:30:37.422448045+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-02T13:30:37.493082045+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-02T13:30:37.497793279+08:00","level":"INFO","msg":"stream: closing","id":"w3wdpl8c"} +{"time":"2026-06-02T13:30:37.497815672+08:00","level":"INFO","msg":"handler: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T13:30:37.497920938+08:00","level":"INFO","msg":"sender: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T13:30:37.497929477+08:00","level":"INFO","msg":"stream: closed","id":"w3wdpl8c"} diff --git a/wandb/run-20260602_132409-w3wdpl8c/logs/debug.log b/wandb/run-20260602_132409-w3wdpl8c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..49e99173fd006f1b3426119698a0007087842b42 --- /dev/null +++ b/wandb/run-20260602_132409-w3wdpl8c/logs/debug.log @@ -0,0 +1,26 @@ +2026-06-02 13:24:09,348 INFO MainThread:750246 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-02 13:24:09,348 INFO MainThread:750246 [wandb_setup.py:_flush():81] Configure stats pid to 750246 +2026-06-02 13:24:09,348 INFO MainThread:750246 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-02 13:24:09,348 INFO MainThread:750246 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/wandb/run-20260602_132409-w3wdpl8c/logs/debug.log +2026-06-02 13:24:09,348 INFO MainThread:750246 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/wandb/run-20260602_132409-w3wdpl8c/logs/debug-internal.log +2026-06-02 13:24:09,348 INFO MainThread:750246 [wandb_init.py:init():844] calling init triggers +2026-06-02 13:24:09,348 INFO MainThread:750246 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'_wandb': {'code_path': 'code/train/rl_lavida.py'}} +2026-06-02 13:24:09,348 INFO MainThread:750246 [wandb_init.py:init():892] starting backend +2026-06-02 13:24:09,594 INFO MainThread:750246 [wandb_init.py:init():895] sending inform_init request +2026-06-02 13:24:09,598 INFO MainThread:750246 [wandb_init.py:init():903] backend started and connected +2026-06-02 13:24:09,599 INFO MainThread:750246 [wandb_init.py:init():973] updated telemetry +2026-06-02 13:24:09,605 INFO MainThread:750246 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-02 13:24:10,412 INFO MainThread:750246 [wandb_init.py:init():1037] run resumed +2026-06-02 13:24:10,416 INFO MainThread:750246 [wandb_init.py:init():1042] starting run threads in backend +2026-06-02 13:24:10,536 INFO MainThread:750246 [wandb_run.py:_console_start():2524] atexit reg +2026-06-02 13:24:10,536 INFO MainThread:750246 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-02 13:24:10,537 INFO MainThread:750246 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-02 13:24:10,537 INFO MainThread:750246 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-02 13:24:10,540 INFO MainThread:750246 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-02 13:24:10,541 INFO MainThread:750246 [wandb_run.py:_config_callback():1403] config_cb None None {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'w3wdpl8c', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_revvideo_orig10k_4gpu_len128', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 10, 'experiment.save_every': 5, 'experiment.eval_every': 5, 'experiment.current_epoch': 1, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'experiment.logging_dir': 'rl_lavida_revvideo_orig10k_4gpu_len128/logs', 'model.pretrained_model': '/data4/runqing/hf/lavida-llada-v1.0-instruct', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_sft_train_cleaned_10k', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.05, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 5e-07, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 5e-07, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy.0': 'low_confidence_static', 'evaluation.remasking_strategy.1': 'low_confidence_dynamic', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 32, 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'config': 'configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml'} +2026-06-02 13:30:37,296 INFO MainThread:750246 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c +2026-06-02 13:30:37,297 INFO MainThread:750246 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-02 13:30:37,297 INFO MainThread:750246 [wandb_run.py:_restore():2472] restore +2026-06-02 13:30:37,297 INFO MainThread:750246 [wandb_run.py:_restore():2478] restore done +2026-06-02 13:30:37,497 INFO MainThread:750246 [wandb_run.py:_footer_sync_info():3868] logging synced files