diff --git a/reward/wandb/run-20260529_194707-o3l1vpla/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260529_194707-o3l1vpla/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260529_194707-o3l1vpla/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260529_194707-o3l1vpla/files/config.yaml b/reward/wandb/run-20260529_194707-o3l1vpla/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9d28cc72985047d42673a4e14fa8373818aec42 --- /dev/null +++ b/reward/wandb/run-20260529_194707-o3l1vpla/files/config.yaml @@ -0,0 +1,273 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + 9pqfh5lrgyl0srsilyb4o7f1khpc7ymu: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=o3l1vpla + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=1 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "623847804928" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-29T11:47:07.400494Z" + writerId: 9pqfh5lrgyl0srsilyb4o7f1khpc7ymu + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 13 + - 14 + - 16 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 1 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: o3l1vpla diff --git a/reward/wandb/run-20260529_194707-o3l1vpla/files/output.log b/reward/wandb/run-20260529_194707-o3l1vpla/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..dc7a10bb3603646fbf8121c622280e801c52ff8c --- /dev/null +++ b/reward/wandb/run-20260529_194707-o3l1vpla/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 1 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.4824 avg length: 126.07 diff --git a/reward/wandb/run-20260529_194707-o3l1vpla/files/requirements.txt b/reward/wandb/run-20260529_194707-o3l1vpla/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260529_194707-o3l1vpla/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-metadata.json b/reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4a0d71e7b2df8e1211ade8f20983c6c77c321e73 --- /dev/null +++ b/reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-29T11:47:07.400494Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=o3l1vpla", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=1" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "623847804928" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "9pqfh5lrgyl0srsilyb4o7f1khpc7ymu" +} \ No newline at end of file diff --git a/reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-summary.json b/reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..cdbfd43f5196975c1a192b9dd6172404993ec9c2 --- /dev/null +++ b/reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/unbiased_bonus":0.07499999999999998,"_wandb":{"runtime":0},"reward/acc":0.482421875,"reward/t_grpo_bonus":0.2765625000000003,"reward/train_step":1,"_timestamp":1.7800552284795003e+09,"_runtime":0.276835887,"_step":1,"reward/avg_response_length":126.07421875} \ No newline at end of file diff --git a/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug-core.log b/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..298439b5a07e02e7709c66273d7ed6415d5f703c --- /dev/null +++ b/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-29T19:47:07.473481181+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpbw7abz65/port-1212640.txt","pid":1212640,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-29T19:47:07.474437183+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1212640-1212740-3778981817/socket","Net":"unix"}} +{"time":"2026-05-29T19:47:07.474559698+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1212640} +{"time":"2026-05-29T19:47:07.655012851+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-29T19:47:07.664220576+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"o3l1vpla","id":"1(@)"} +{"time":"2026-05-29T19:47:07.947077344+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"o3l1vpla","id":"1(@)"} +{"time":"2026-05-29T19:47:08.483101048+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"lu9yljxzchbl"} +{"time":"2026-05-29T19:47:08.683766011+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"lu9yljxzchbl"} +{"time":"2026-05-29T19:47:08.685066243+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"o3l1vpla","id":"1(@)"} +{"time":"2026-05-29T19:47:08.685504424+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"o3l1vpla","id":"1(@)"} +{"time":"2026-05-29T19:47:08.685645737+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-29T19:47:08.685676362+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-29T19:47:08.685684169+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-29T19:47:08.685710476+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-29T19:47:08.685743906+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1212640-1212740-3778981817/socket","Net":"unix"}} +{"time":"2026-05-29T19:47:08.685770189+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-29T19:47:08.68577738+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-29T19:47:08.685784402+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug-internal.log b/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e6aa95bc911f305bf56e76e7f267dc50276b1000 --- /dev/null +++ b/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug-internal.log @@ -0,0 +1,27 @@ +{"time":"2026-05-29T19:47:07.664329293+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-29T19:47:07.946806555+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-29T19:47:07.946916582+08:00","level":"INFO","msg":"stream: created new stream","id":"o3l1vpla"} +{"time":"2026-05-29T19:47:07.946954411+08:00","level":"INFO","msg":"handler: started","stream_id":"o3l1vpla"} +{"time":"2026-05-29T19:47:07.94706896+08:00","level":"INFO","msg":"stream: started","id":"o3l1vpla"} +{"time":"2026-05-29T19:47:07.947088794+08:00","level":"INFO","msg":"writer: started","stream_id":"o3l1vpla"} +{"time":"2026-05-29T19:47:07.947090844+08:00","level":"INFO","msg":"sender: started","stream_id":"o3l1vpla"} +{"time":"2026-05-29T19:47:07.948297319+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-29T19:47:08.206739239+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-29T19:47:08.62874404+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a47ee7cc56a7d98ad3f50a5a4c0649595b715b64e89e47637ac5aff422ec7383"} +{"time":"2026-05-29T19:47:08.628809977+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=883c4209c5d742770b3a6d59ac71abe2715046eb59ec69dccaa1a5b567ba579b"} +{"time":"2026-05-29T19:47:08.628827792+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-metadata.json18B406CC9D55826317e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a47ee7cc56a7d98ad3f50a5a4c0649595b715b64e89e47637ac5aff422ec7383, Size: 2849}"} +{"time":"2026-05-29T19:47:08.628868986+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/o3l1vpla/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/config.yaml18B406CC9F0C3E1B29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=883c4209c5d742770b3a6d59ac71abe2715046eb59ec69dccaa1a5b567ba579b, Size: 7755}"} +{"time":"2026-05-29T19:47:08.630071945+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cde00db00b44971fb11a4d87808dfbc4eeeac5cbe9873b1fa83739955ab417da"} +{"time":"2026-05-29T19:47:08.630148543+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/o3l1vpla/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/output.log18B406CC9F05F21929678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cde00db00b44971fb11a4d87808dfbc4eeeac5cbe9873b1fa83739955ab417da, Size: 110}"} +{"time":"2026-05-29T19:47:08.63086663+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e4bb8822ef29df26949ff5ddef85264847f7f4b05c641a324b91d1585449ba55"} +{"time":"2026-05-29T19:47:08.630961997+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/o3l1vpla/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/requirements.txt18B406CC9DD7FC864b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e4bb8822ef29df26949ff5ddef85264847f7f4b05c641a324b91d1585449ba55, Size: 2982}"} +{"time":"2026-05-29T19:47:08.633145978+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7226b6bd23e4ef9db8e48e70a2774674ebb0cbec0bb484247d588b1c4633fbc5"} +{"time":"2026-05-29T19:47:08.633266177+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-summary.json18B406CC9F223DB329678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7226b6bd23e4ef9db8e48e70a2774674ebb0cbec0bb484247d588b1c4633fbc5, Size: 267}"} +{"time":"2026-05-29T19:47:08.633747103+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=03e385aef9c6b56540adbe0ddb3f51b263ba0e7776fdc7debbb663e00aa220b0"} +{"time":"2026-05-29T19:47:08.633803977+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/o3l1vpla/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/code/reward/rl_reward_video_reversal.py18B406CC9F05F1CD29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=03e385aef9c6b56540adbe0ddb3f51b263ba0e7776fdc7debbb663e00aa220b0, Size: 14790}"} +{"time":"2026-05-29T19:47:08.63416604+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-29T19:47:08.676087931+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-29T19:47:08.685079701+08:00","level":"INFO","msg":"stream: closing","id":"o3l1vpla"} +{"time":"2026-05-29T19:47:08.685100599+08:00","level":"INFO","msg":"handler: closed","stream_id":"o3l1vpla"} +{"time":"2026-05-29T19:47:08.685170573+08:00","level":"INFO","msg":"sender: closed","stream_id":"o3l1vpla"} +{"time":"2026-05-29T19:47:08.685177465+08:00","level":"INFO","msg":"stream: closed","id":"o3l1vpla"} diff --git a/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug.log b/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f71eb7a5436d6ad2409d46929ce30e8faf39a44b --- /dev/null +++ b/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug.log @@ -0,0 +1,24 @@ +2026-05-29 19:47:07,401 INFO MainThread:1212640 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-29 19:47:07,402 INFO MainThread:1212640 [wandb_setup.py:_flush():81] Configure stats pid to 1212640 +2026-05-29 19:47:07,402 INFO MainThread:1212640 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-29 19:47:07,402 INFO MainThread:1212640 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug.log +2026-05-29 19:47:07,402 INFO MainThread:1212640 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug-internal.log +2026-05-29 19:47:07,402 INFO MainThread:1212640 [wandb_init.py:init():844] calling init triggers +2026-05-29 19:47:07,402 INFO MainThread:1212640 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'o3l1vpla', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 1, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-29 19:47:07,402 INFO MainThread:1212640 [wandb_init.py:init():892] starting backend +2026-05-29 19:47:07,656 INFO MainThread:1212640 [wandb_init.py:init():895] sending inform_init request +2026-05-29 19:47:07,660 INFO MainThread:1212640 [wandb_init.py:init():903] backend started and connected +2026-05-29 19:47:07,662 INFO MainThread:1212640 [wandb_init.py:init():973] updated telemetry +2026-05-29 19:47:07,673 INFO MainThread:1212640 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-29 19:47:08,204 INFO MainThread:1212640 [wandb_init.py:init():1042] starting run threads in backend +2026-05-29 19:47:08,315 INFO MainThread:1212640 [wandb_run.py:_console_start():2524] atexit reg +2026-05-29 19:47:08,316 INFO MainThread:1212640 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-29 19:47:08,316 INFO MainThread:1212640 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-29 19:47:08,316 INFO MainThread:1212640 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-29 19:47:08,318 INFO MainThread:1212640 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-29 19:47:08,479 INFO MainThread:1212640 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/o3l1vpla +2026-05-29 19:47:08,480 INFO MainThread:1212640 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-29 19:47:08,480 INFO MainThread:1212640 [wandb_run.py:_restore():2472] restore +2026-05-29 19:47:08,480 INFO MainThread:1212640 [wandb_run.py:_restore():2478] restore done +2026-05-29 19:47:08,684 INFO MainThread:1212640 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260529_194707-o3l1vpla/run-o3l1vpla.wandb b/reward/wandb/run-20260529_194707-o3l1vpla/run-o3l1vpla.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1de21dbf507290ad49c7c5292fbba6067abe99d2 Binary files /dev/null and b/reward/wandb/run-20260529_194707-o3l1vpla/run-o3l1vpla.wandb differ diff --git a/reward/wandb/run-20260529_221548-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260529_221548-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ee0b5a98f3fdcf5776eb09ac4eeb007093c73b19 --- /dev/null +++ b/reward/wandb/run-20260529_221548-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-29T14:15:48.972009Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=2" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "624733896704" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "of7uuhm8djcrujnrkdjlueoxbofh7yt3" +} \ No newline at end of file diff --git a/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..adea845f61bc7f19a66c2f78f80db2e550aaf230 --- /dev/null +++ b/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-29T22:15:49.038496026+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpextmq8yv/port-1544888.txt","pid":1544888,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-29T22:15:49.039517557+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1544888} +{"time":"2026-05-29T22:15:49.039525747+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1544888-1545045-3458517359/socket","Net":"unix"}} +{"time":"2026-05-29T22:15:49.220936217+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-29T22:15:49.231304816+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-29T22:15:49.532027363+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-29T22:15:49.887995417+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"zs4zg8uz72tg"} +{"time":"2026-05-29T22:15:50.066303944+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"zs4zg8uz72tg"} +{"time":"2026-05-29T22:15:50.067591053+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-29T22:15:50.068479156+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-29T22:15:50.068513746+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-29T22:15:50.068538601+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-29T22:15:50.068549089+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-29T22:15:50.068566656+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-29T22:15:50.068599467+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-29T22:15:50.068611716+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-29T22:15:50.06874089+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1544888-1545045-3458517359/socket","Net":"unix"}} +{"time":"2026-05-29T22:15:50.068796137+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b2706257ae4e361940dcabe72f110d61d1ef04d0 --- /dev/null +++ b/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-29T22:15:49.231529459+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-29T22:15:49.531421968+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-29T22:15:49.531682346+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-29T22:15:49.53183915+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-29T22:15:49.532000568+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-29T22:15:49.532047358+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-29T22:15:49.532051598+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-29T22:15:49.534503641+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-29T22:15:49.676991563+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-29T22:15:49.886864995+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":2,"current":23} +{"time":"2026-05-29T22:15:50.012976753+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1c618949f54f91ead5f029d477b6f3a66b28049d8038323dc4e7c19987324b9d"} +{"time":"2026-05-29T22:15:50.013148766+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B40EE9CACCD1E729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1c618949f54f91ead5f029d477b6f3a66b28049d8038323dc4e7c19987324b9d, Size: 339}"} +{"time":"2026-05-29T22:15:50.013183236+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c615766f40662e6943a47d0789f9209b0dce3b6b820193907bbc7ec403b423a2"} +{"time":"2026-05-29T22:15:50.013319357+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B40EE9CAC0BA7229678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c615766f40662e6943a47d0789f9209b0dce3b6b820193907bbc7ec403b423a2, Size: 7994}"} +{"time":"2026-05-29T22:15:50.013842455+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9e76d2be777d1c7ca83aad7eb2ea22d1cbf6d583c423df7e1d096b094110beb1"} +{"time":"2026-05-29T22:15:50.013942597+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B40EE9CACDD07029678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9e76d2be777d1c7ca83aad7eb2ea22d1cbf6d583c423df7e1d096b094110beb1, Size: 2849}"} +{"time":"2026-05-29T22:15:50.014336234+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e6a49b3ce09eab10673811108dd02727a560b4aa9b50821a1ce992f363b5e7f7"} +{"time":"2026-05-29T22:15:50.014497174+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B40EE9CB5CF82117e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e6a49b3ce09eab10673811108dd02727a560b4aa9b50821a1ce992f363b5e7f7, Size: 2982}"} +{"time":"2026-05-29T22:15:50.015699606+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=eefbe5c2c8d9371e1c63c8ec320a10cf528f816dec9c2f86a97a3194a7363fe0"} +{"time":"2026-05-29T22:15:50.015867408+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B40EE9CB77AB3817e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=eefbe5c2c8d9371e1c63c8ec320a10cf528f816dec9c2f86a97a3194a7363fe0, Size: 110}"} +{"time":"2026-05-29T22:15:50.022155163+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9d1064af61d48bce0c860f3c3c26be81a940a20ec21aab3db9f87403d7af47e4"} +{"time":"2026-05-29T22:15:50.022205902+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B40EE9CB5CA45917e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9d1064af61d48bce0c860f3c3c26be81a940a20ec21aab3db9f87403d7af47e4, Size: 14790}"} +{"time":"2026-05-29T22:15:50.022643716+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-29T22:15:50.062268078+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-29T22:15:50.067618841+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-29T22:15:50.067638466+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-29T22:15:50.067710417+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-29T22:15:50.06774884+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a48e9f956997b48c2ae1f5924bf698b1e02ebf18 --- /dev/null +++ b/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-29 22:15:48,974 INFO MainThread:1544888 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-29 22:15:48,974 INFO MainThread:1544888 [wandb_setup.py:_flush():81] Configure stats pid to 1544888 +2026-05-29 22:15:48,974 INFO MainThread:1544888 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-29 22:15:48,975 INFO MainThread:1544888 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug.log +2026-05-29 22:15:48,975 INFO MainThread:1544888 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug-internal.log +2026-05-29 22:15:48,975 INFO MainThread:1544888 [wandb_init.py:init():844] calling init triggers +2026-05-29 22:15:48,975 INFO MainThread:1544888 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 2, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-29 22:15:48,975 INFO MainThread:1544888 [wandb_init.py:init():892] starting backend +2026-05-29 22:15:49,221 INFO MainThread:1544888 [wandb_init.py:init():895] sending inform_init request +2026-05-29 22:15:49,227 INFO MainThread:1544888 [wandb_init.py:init():903] backend started and connected +2026-05-29 22:15:49,228 INFO MainThread:1544888 [wandb_init.py:init():973] updated telemetry +2026-05-29 22:15:49,234 INFO MainThread:1544888 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-29 22:15:49,670 INFO MainThread:1544888 [wandb_init.py:init():1037] run resumed +2026-05-29 22:15:49,674 INFO MainThread:1544888 [wandb_init.py:init():1042] starting run threads in backend +2026-05-29 22:15:49,764 INFO MainThread:1544888 [wandb_run.py:_console_start():2524] atexit reg +2026-05-29 22:15:49,765 INFO MainThread:1544888 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-29 22:15:49,765 INFO MainThread:1544888 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-29 22:15:49,765 INFO MainThread:1544888 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-29 22:15:49,769 INFO MainThread:1544888 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-29 22:15:49,886 INFO MainThread:1544888 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-29 22:15:49,887 INFO MainThread:1544888 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-29 22:15:49,887 INFO MainThread:1544888 [wandb_run.py:_restore():2472] restore +2026-05-29 22:15:49,887 INFO MainThread:1544888 [wandb_run.py:_restore():2478] restore done +2026-05-29 22:15:50,066 INFO MainThread:1544888 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260529_221548-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260529_221548-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..037a25391d53df556f5d2218cb19789641cd885e Binary files /dev/null and b/reward/wandb/run-20260529_221548-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_001305-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_001305-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_001305-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_001305-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_001305-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c75401913f1be31733a2e4394199d509b62cc534 --- /dev/null +++ b/reward/wandb/run-20260530_001305-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + nbjlrwv1znq9ynwn7j97kr9f61plnzak: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=4 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625472454656" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-29T16:13:05.220431Z" + writerId: nbjlrwv1znq9ynwn7j97kr9f61plnzak + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 4 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_001305-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_001305-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c6c3f90b4fab1e89fa3b2b8cf261dcef65af3bb6 --- /dev/null +++ b/reward/wandb/run-20260530_001305-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 4 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.4590 avg length: 126.43 diff --git a/reward/wandb/run-20260530_001305-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_001305-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_001305-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b2aaf225fb011cdaf3b661a9c5b6ab6800d90658 --- /dev/null +++ b/reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-29T16:13:05.220431Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=4" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625472454656" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "nbjlrwv1znq9ynwn7j97kr9f61plnzak" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fb6eaf39544157fe562f27b9f04cdb0960edc96e --- /dev/null +++ b/reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":2202,"reward/t_grpo_bonus":0.2718750000000003,"reward/unbiased_bonus":0.09374999999999996,"train/learning_rate":7.300325188655752e-07,"reward/acc":0.396484375,"_step":22,"train/loss":0.3783190548419952,"reward/train_step":1,"reward/avg_response_length":124.568359375,"_wandb":{"runtime":2202},"_timestamp":1.7800614303455548e+09} \ No newline at end of file diff --git a/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..02e4280d8c5e874d9d3f7b427e7c08f882b0da85 --- /dev/null +++ b/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T00:13:05.286065679+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpp62euswa/port-1751242.txt","pid":1751242,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T00:13:05.287189973+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1751242} +{"time":"2026-05-30T00:13:05.287151511+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1751242-1751409-2046619474/socket","Net":"unix"}} +{"time":"2026-05-30T00:13:05.468769376+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T00:13:05.477564719+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T00:13:05.780034643+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T00:13:06.242278206+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"k7igfwpnqhw4"} +{"time":"2026-05-30T00:13:06.425328395+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"k7igfwpnqhw4"} +{"time":"2026-05-30T00:13:06.42595584+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T00:13:06.42694024+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T00:13:06.426952049+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T00:13:06.426964821+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T00:13:06.426975472+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T00:13:06.426985269+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T00:13:06.427015094+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T00:13:06.427038141+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T00:13:06.427156462+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1751242-1751409-2046619474/socket","Net":"unix"}} +{"time":"2026-05-30T00:13:06.427212116+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..abc4a64fd7694d6e29632ad1ce6f4c38065d3081 --- /dev/null +++ b/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T00:13:05.477770765+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T00:13:05.779450034+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T00:13:05.779652585+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T00:13:05.779744219+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T00:13:05.780014388+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T00:13:05.780055478+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T00:13:05.780055506+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T00:13:05.782697905+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T00:13:06.028313692+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T00:13:06.2385663+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":4,"current":23} +{"time":"2026-05-30T00:13:06.368370958+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8578fc7b87857bdd751003ba69247abe51fa215fdca5c83876ba57e01529f06f"} +{"time":"2026-05-30T00:13:06.368497869+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B415501297BED117e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8578fc7b87857bdd751003ba69247abe51fa215fdca5c83876ba57e01529f06f, Size: 110}"} +{"time":"2026-05-30T00:13:06.368902337+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=74595d174d88f5f42545e11c808630835e87001e8a5296f7c2c718f2d4995bdc"} +{"time":"2026-05-30T00:13:06.369039843+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B415501291CB6117e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=74595d174d88f5f42545e11c808630835e87001e8a5296f7c2c718f2d4995bdc, Size: 7994}"} +{"time":"2026-05-30T00:13:06.369639945+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a2d0a58bb258141ad482f96245ac2182bdfbbdab1e2fa96a04db7ecbc47d2843"} +{"time":"2026-05-30T00:13:06.369798939+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B415501216918429678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a2d0a58bb258141ad482f96245ac2182bdfbbdab1e2fa96a04db7ecbc47d2843, Size: 2849}"} +{"time":"2026-05-30T00:13:06.370661611+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=99480e690e5934203d2d0ba772e2d0ce0a39c37c4d8b6b6d9db3c4482f0b9fc1"} +{"time":"2026-05-30T00:13:06.370783078+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B41550110447D34b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=99480e690e5934203d2d0ba772e2d0ce0a39c37c4d8b6b6d9db3c4482f0b9fc1, Size: 341}"} +{"time":"2026-05-30T00:13:06.370857593+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=932abcfbee1a259382384175d32ca10d553bb10e5e18f491f305f95f6f890ab1"} +{"time":"2026-05-30T00:13:06.370975128+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B41550122B1FAB29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=932abcfbee1a259382384175d32ca10d553bb10e5e18f491f305f95f6f890ab1, Size: 2982}"} +{"time":"2026-05-30T00:13:06.378523807+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce774c4d207911537ae5971450542027f4d47ab73f9f0e5a233371ed80539cb7"} +{"time":"2026-05-30T00:13:06.378678161+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B4155010FA7E6E4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce774c4d207911537ae5971450542027f4d47ab73f9f0e5a233371ed80539cb7, Size: 14790}"} +{"time":"2026-05-30T00:13:06.378940351+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T00:13:06.421106068+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T00:13:06.425978529+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T00:13:06.425995378+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T00:13:06.426060973+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T00:13:06.426080026+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2530ad9cf70e5c62c62a7a9ce117784884370098 --- /dev/null +++ b/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 00:13:05,223 INFO MainThread:1751242 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 00:13:05,223 INFO MainThread:1751242 [wandb_setup.py:_flush():81] Configure stats pid to 1751242 +2026-05-30 00:13:05,223 INFO MainThread:1751242 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 00:13:05,223 INFO MainThread:1751242 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug.log +2026-05-30 00:13:05,223 INFO MainThread:1751242 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug-internal.log +2026-05-30 00:13:05,223 INFO MainThread:1751242 [wandb_init.py:init():844] calling init triggers +2026-05-30 00:13:05,223 INFO MainThread:1751242 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 4, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 00:13:05,223 INFO MainThread:1751242 [wandb_init.py:init():892] starting backend +2026-05-30 00:13:05,469 INFO MainThread:1751242 [wandb_init.py:init():895] sending inform_init request +2026-05-30 00:13:05,474 INFO MainThread:1751242 [wandb_init.py:init():903] backend started and connected +2026-05-30 00:13:05,475 INFO MainThread:1751242 [wandb_init.py:init():973] updated telemetry +2026-05-30 00:13:05,479 INFO MainThread:1751242 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 00:13:06,023 INFO MainThread:1751242 [wandb_init.py:init():1037] run resumed +2026-05-30 00:13:06,026 INFO MainThread:1751242 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 00:13:06,116 INFO MainThread:1751242 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 00:13:06,116 INFO MainThread:1751242 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 00:13:06,116 INFO MainThread:1751242 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 00:13:06,116 INFO MainThread:1751242 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 00:13:06,120 INFO MainThread:1751242 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 00:13:06,238 INFO MainThread:1751242 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 00:13:06,238 INFO MainThread:1751242 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 00:13:06,239 INFO MainThread:1751242 [wandb_run.py:_restore():2472] restore +2026-05-30 00:13:06,239 INFO MainThread:1751242 [wandb_run.py:_restore():2478] restore done +2026-05-30 00:13:06,425 INFO MainThread:1751242 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_001305-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_001305-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f453bb811e45e1fad70e30065523dc7b92b0db47 Binary files /dev/null and b/reward/wandb/run-20260530_001305-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_022315-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_022315-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_022315-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_022315-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_022315-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c6225842a9c3075b9573c5b2d64a50b87f90b17 --- /dev/null +++ b/reward/wandb/run-20260530_022315-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + rit67u7afan4qam30286z3mhxid958av: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=6 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625479712768" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-29T18:23:15.496253Z" + writerId: rit67u7afan4qam30286z3mhxid958av + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 6 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_022315-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_022315-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..22a63f722d10a5341c866cc30f18117360db5b72 --- /dev/null +++ b/reward/wandb/run-20260530_022315-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 6 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.3926 avg length: 126.23 diff --git a/reward/wandb/run-20260530_022315-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_022315-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_022315-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..717045cb318729380e5576fe645a218c4a66f3f1 --- /dev/null +++ b/reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-29T18:23:15.496253Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=6" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625479712768" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "rit67u7afan4qam30286z3mhxid958av" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..49e1f94c579f3e4e79f1b98777cfc4cde0ca472e --- /dev/null +++ b/reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"_timestamp":1.7800760271916904e+09,"reward/train_step":1,"_step":26,"_wandb":{"runtime":4222},"train/learning_rate":7.985792958513929e-07,"reward/t_grpo_bonus":0.2718750000000003,"train/loss":-0.37430867552757263,"reward/avg_response_length":124.568359375,"reward/unbiased_bonus":0.09374999999999996,"reward/acc":0.396484375,"_runtime":4222} \ No newline at end of file diff --git a/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..6e80dc962885a3c0c74057b25f5f4645c1c4c803 --- /dev/null +++ b/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T02:23:15.564585624+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpvrnq85bv/port-1972975.txt","pid":1972975,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T02:23:15.567024905+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1972975} +{"time":"2026-05-30T02:23:15.56702786+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1972975-1973131-39973928/socket","Net":"unix"}} +{"time":"2026-05-30T02:23:15.745166446+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T02:23:15.755860291+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T02:23:16.055380273+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T02:23:16.546440823+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fu9pap2v31pp"} +{"time":"2026-05-30T02:23:16.730663414+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fu9pap2v31pp"} +{"time":"2026-05-30T02:23:16.732391613+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T02:23:16.733226984+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T02:23:16.733240787+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T02:23:16.733255106+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T02:23:16.733267833+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T02:23:16.733276961+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T02:23:16.733350086+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1972975-1973131-39973928/socket","Net":"unix"}} +{"time":"2026-05-30T02:23:16.733430555+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T02:23:16.733447157+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T02:23:16.733468785+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d98597a7de2be9d5e93dd67ae275b2c0b8d491cd --- /dev/null +++ b/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T02:23:15.756016562+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T02:23:16.054970322+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T02:23:16.055173873+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T02:23:16.055246728+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T02:23:16.055363043+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T02:23:16.055380438+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T02:23:16.055393379+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T02:23:16.059603255+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T02:23:16.335493869+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T02:23:16.545805834+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":6,"current":27} +{"time":"2026-05-30T02:23:16.677526751+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=821bfbf28a105fa324036c7aab937e5955c74cc8ce99454b4e05e18681bed874"} +{"time":"2026-05-30T02:23:16.677717912+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B41C6A8D4048084b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=821bfbf28a105fa324036c7aab937e5955c74cc8ce99454b4e05e18681bed874, Size: 7994}"} +{"time":"2026-05-30T02:23:16.678145344+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9936bf6fb1a6034661afb45fccd887766fa50da78ed3dd0dd0a9c75ac6738e28"} +{"time":"2026-05-30T02:23:16.67830001+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B41C6A8C90601F29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9936bf6fb1a6034661afb45fccd887766fa50da78ed3dd0dd0a9c75ac6738e28, Size: 343}"} +{"time":"2026-05-30T02:23:16.679660464+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d1a151f9161d8724a96f5cf5290917f0f27f95e56c278d8aeb0bef59702e3624"} +{"time":"2026-05-30T02:23:16.679851441+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B41C6A8D3A116E17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d1a151f9161d8724a96f5cf5290917f0f27f95e56c278d8aeb0bef59702e3624, Size: 110}"} +{"time":"2026-05-30T02:23:16.68138543+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=63d9ebec3584b60becb67d7402bcda98014208276ed41ff4039e920fa99f4335"} +{"time":"2026-05-30T02:23:16.681533107+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B41C6A8D37E2C717e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=63d9ebec3584b60becb67d7402bcda98014208276ed41ff4039e920fa99f4335, Size: 2849}"} +{"time":"2026-05-30T02:23:16.685224569+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7f134f6231481934027ed53fe2c3b34d2356fabc7496ac5ed8bf95a505cf10e5"} +{"time":"2026-05-30T02:23:16.685330732+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B41C6A8C9F174129678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7f134f6231481934027ed53fe2c3b34d2356fabc7496ac5ed8bf95a505cf10e5, Size: 14790}"} +{"time":"2026-05-30T02:23:16.686129396+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=33b21b6b3a8f6747127fcbaccf98aee64e2a8e82b3fbfe76e620d358ad90e198"} +{"time":"2026-05-30T02:23:16.68618286+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B41C6A8CF03B7E29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=33b21b6b3a8f6747127fcbaccf98aee64e2a8e82b3fbfe76e620d358ad90e198, Size: 2982}"} +{"time":"2026-05-30T02:23:16.686494398+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T02:23:16.727909569+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T02:23:16.732429923+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T02:23:16.732456013+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T02:23:16.732531196+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T02:23:16.732548218+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..57ddff6c657cab5886f1a7e9815608df9cbbe72d --- /dev/null +++ b/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 02:23:15,499 INFO MainThread:1972975 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 02:23:15,499 INFO MainThread:1972975 [wandb_setup.py:_flush():81] Configure stats pid to 1972975 +2026-05-30 02:23:15,499 INFO MainThread:1972975 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 02:23:15,499 INFO MainThread:1972975 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug.log +2026-05-30 02:23:15,499 INFO MainThread:1972975 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug-internal.log +2026-05-30 02:23:15,499 INFO MainThread:1972975 [wandb_init.py:init():844] calling init triggers +2026-05-30 02:23:15,499 INFO MainThread:1972975 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 6, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 02:23:15,499 INFO MainThread:1972975 [wandb_init.py:init():892] starting backend +2026-05-30 02:23:15,745 INFO MainThread:1972975 [wandb_init.py:init():895] sending inform_init request +2026-05-30 02:23:15,751 INFO MainThread:1972975 [wandb_init.py:init():903] backend started and connected +2026-05-30 02:23:15,753 INFO MainThread:1972975 [wandb_init.py:init():973] updated telemetry +2026-05-30 02:23:15,756 INFO MainThread:1972975 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 02:23:16,328 INFO MainThread:1972975 [wandb_init.py:init():1037] run resumed +2026-05-30 02:23:16,333 INFO MainThread:1972975 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 02:23:16,425 INFO MainThread:1972975 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 02:23:16,425 INFO MainThread:1972975 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 02:23:16,425 INFO MainThread:1972975 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 02:23:16,425 INFO MainThread:1972975 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 02:23:16,428 INFO MainThread:1972975 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 02:23:16,545 INFO MainThread:1972975 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 02:23:16,545 INFO MainThread:1972975 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 02:23:16,546 INFO MainThread:1972975 [wandb_run.py:_restore():2472] restore +2026-05-30 02:23:16,546 INFO MainThread:1972975 [wandb_run.py:_restore():2478] restore done +2026-05-30 02:23:16,731 INFO MainThread:1972975 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_022315-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_022315-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..109083db0ced296dbd0ca5721ad3bb0de9c8bf73 Binary files /dev/null and b/reward/wandb/run-20260530_022315-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_042845-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_042845-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_042845-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_042845-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_042845-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bb2c22b9ae3595392e848898f9a92112d438d04 --- /dev/null +++ b/reward/wandb/run-20260530_042845-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + iv8zvsdp3bk0inm3f71d72crajbgfbvu: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=8 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625480916992" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-29T20:28:45.290476Z" + writerId: iv8zvsdp3bk0inm3f71d72crajbgfbvu + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 8 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_042845-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_042845-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..143ed73bf3974799af73cb0796ccd9302281a7df --- /dev/null +++ b/reward/wandb/run-20260530_042845-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 8 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.4688 avg length: 126.02 diff --git a/reward/wandb/run-20260530_042845-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_042845-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_042845-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..67b0a696609fc5de5c9f43000da6eb8a48271f6d --- /dev/null +++ b/reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-29T20:28:45.290476Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=8" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625480916992" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "iv8zvsdp3bk0inm3f71d72crajbgfbvu" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fd1cc2d84fbb5653827d684c243f9c374cb1d58d --- /dev/null +++ b/reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/acc":0.396484375,"_runtime":5992,"reward/avg_response_length":124.568359375,"train/loss":-0.37430867552757263,"reward/train_step":1,"_timestamp":1.7800760271916904e+09,"_wandb":{"runtime":5992},"reward/t_grpo_bonus":0.2718750000000003,"train/learning_rate":7.985792958513929e-07,"reward/unbiased_bonus":0.09374999999999996,"_step":26} \ No newline at end of file diff --git a/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..1db874c32d8dc7c1e0c328e933b1f9386cbde6c5 --- /dev/null +++ b/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T04:28:45.357187774+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpe2ektbof/port-2189265.txt","pid":2189265,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T04:28:45.357749747+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2189265} +{"time":"2026-05-30T04:28:45.357737063+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2189265-2189429-3673607564/socket","Net":"unix"}} +{"time":"2026-05-30T04:28:45.538534603+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T04:28:45.547157906+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T04:28:45.852526644+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T04:28:46.370665657+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"viw1ypxpd56s"} +{"time":"2026-05-30T04:28:46.550283848+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"viw1ypxpd56s"} +{"time":"2026-05-30T04:28:46.551572545+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T04:28:46.552278813+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T04:28:46.552287358+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T04:28:46.552297327+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T04:28:46.552305555+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T04:28:46.552320535+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T04:28:46.552335675+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T04:28:46.552423726+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T04:28:46.552482901+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2189265-2189429-3673607564/socket","Net":"unix"}} +{"time":"2026-05-30T04:28:46.552538115+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d4e112872960ee71c89f106f23066943c08f4ff0 --- /dev/null +++ b/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T04:28:45.547363162+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T04:28:45.851985924+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T04:28:45.852256915+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T04:28:45.852386761+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T04:28:45.852507919+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T04:28:45.852536962+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T04:28:45.852534157+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T04:28:45.858252069+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T04:28:46.158666066+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T04:28:46.369997093+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":8,"current":27} +{"time":"2026-05-30T04:28:46.501935127+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=dadf76cf88cfe1ec58e2105e56b323d6ac39772d8e5bb0ad7ed85e429bd4e85e"} +{"time":"2026-05-30T04:28:46.502060021+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B42343B965E10817e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=dadf76cf88cfe1ec58e2105e56b323d6ac39772d8e5bb0ad7ed85e429bd4e85e, Size: 110}"} +{"time":"2026-05-30T04:28:46.502198522+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6aa16e413100df81d7b2c203db0e968d616ffff4e373204c41041478c7fbfe13"} +{"time":"2026-05-30T04:28:46.502298169+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e05b7e592252a0fe524561bedbb211f5123fe01e58341c518a6f90f8f8ccad28"} +{"time":"2026-05-30T04:28:46.50236231+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B42343B966889E17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6aa16e413100df81d7b2c203db0e968d616ffff4e373204c41041478c7fbfe13, Size: 7994}"} +{"time":"2026-05-30T04:28:46.502536921+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B42343B9A1978F4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e05b7e592252a0fe524561bedbb211f5123fe01e58341c518a6f90f8f8ccad28, Size: 2849}"} +{"time":"2026-05-30T04:28:46.5035542+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=38aaaf3093af2965c378a7cf0d77ea697514c7df7c85d2d318d1ae08d7932f11"} +{"time":"2026-05-30T04:28:46.503761198+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B42343B8DC768729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=38aaaf3093af2965c378a7cf0d77ea697514c7df7c85d2d318d1ae08d7932f11, Size: 343}"} +{"time":"2026-05-30T04:28:46.505924826+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=326cb6c5ef32f7ec9d65c1165435635a9e1d486c435f164c34ec192c8dcdbd4d"} +{"time":"2026-05-30T04:28:46.506020205+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B42343B9BD0B614b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=326cb6c5ef32f7ec9d65c1165435635a9e1d486c435f164c34ec192c8dcdbd4d, Size: 2982}"} +{"time":"2026-05-30T04:28:46.506408954+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=fa24244b919ee0c3b1debd877157ca47f4dfaf89a9c2928302cae995ae265dc6"} +{"time":"2026-05-30T04:28:46.506468235+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B42343B958772917e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=fa24244b919ee0c3b1debd877157ca47f4dfaf89a9c2928302cae995ae265dc6, Size: 14790}"} +{"time":"2026-05-30T04:28:46.506804552+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T04:28:46.547050766+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T04:28:46.551595229+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T04:28:46.551609585+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T04:28:46.551670861+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T04:28:46.551687822+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8e348c56f48d6e35ab5faea9ccae3d575e57246c --- /dev/null +++ b/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 04:28:45,293 INFO MainThread:2189265 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 04:28:45,293 INFO MainThread:2189265 [wandb_setup.py:_flush():81] Configure stats pid to 2189265 +2026-05-30 04:28:45,293 INFO MainThread:2189265 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 04:28:45,293 INFO MainThread:2189265 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug.log +2026-05-30 04:28:45,293 INFO MainThread:2189265 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug-internal.log +2026-05-30 04:28:45,293 INFO MainThread:2189265 [wandb_init.py:init():844] calling init triggers +2026-05-30 04:28:45,293 INFO MainThread:2189265 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 8, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 04:28:45,293 INFO MainThread:2189265 [wandb_init.py:init():892] starting backend +2026-05-30 04:28:45,538 INFO MainThread:2189265 [wandb_init.py:init():895] sending inform_init request +2026-05-30 04:28:45,543 INFO MainThread:2189265 [wandb_init.py:init():903] backend started and connected +2026-05-30 04:28:45,545 INFO MainThread:2189265 [wandb_init.py:init():973] updated telemetry +2026-05-30 04:28:45,548 INFO MainThread:2189265 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 04:28:46,152 INFO MainThread:2189265 [wandb_init.py:init():1037] run resumed +2026-05-30 04:28:46,157 INFO MainThread:2189265 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 04:28:46,246 INFO MainThread:2189265 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 04:28:46,246 INFO MainThread:2189265 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 04:28:46,246 INFO MainThread:2189265 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 04:28:46,246 INFO MainThread:2189265 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 04:28:46,250 INFO MainThread:2189265 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 04:28:46,369 INFO MainThread:2189265 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 04:28:46,370 INFO MainThread:2189265 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 04:28:46,370 INFO MainThread:2189265 [wandb_run.py:_restore():2472] restore +2026-05-30 04:28:46,370 INFO MainThread:2189265 [wandb_run.py:_restore():2478] restore done +2026-05-30 04:28:46,550 INFO MainThread:2189265 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_042845-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_042845-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..6a6535e586eea5126c5d106a7afd9841099391a8 Binary files /dev/null and b/reward/wandb/run-20260530_042845-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_063905-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_063905-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_063905-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_063905-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_063905-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6985d2f891dc502c2daebc65f74b10db4e189eb2 --- /dev/null +++ b/reward/wandb/run-20260530_063905-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + cabjrf3uennqcbtqsnxq22932fx3o8bo: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=10 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625483096064" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-29T22:39:05.986619Z" + writerId: cabjrf3uennqcbtqsnxq22932fx3o8bo + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 10 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_063905-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_063905-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5627c7ef5fc72c4661d3116ab264d566978f13fa --- /dev/null +++ b/reward/wandb/run-20260530_063905-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 10 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.5078 avg length: 125.81 diff --git a/reward/wandb/run-20260530_063905-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_063905-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_063905-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_063905-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_063905-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..40cd563fc15999848faa05e90dae1ef4b33fab87 --- /dev/null +++ b/reward/wandb/run-20260530_063905-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-29T22:39:05.986619Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=10" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625483096064" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "cabjrf3uennqcbtqsnxq22932fx3o8bo" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_063905-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_063905-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bcd2d6b8fbfd03a757c31618224227d4244ee054 --- /dev/null +++ b/reward/wandb/run-20260530_063905-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/unbiased_bonus":0.09374999999999996,"reward/train_step":1,"_step":26,"reward/t_grpo_bonus":0.2718750000000003,"reward/avg_response_length":124.568359375,"_timestamp":1.7800760271916904e+09,"_runtime":8003,"reward/acc":0.396484375,"train/learning_rate":7.985792958513929e-07,"_wandb":{"runtime":8003},"train/loss":-0.37430867552757263} \ No newline at end of file diff --git a/reward/wandb/run-20260530_063905-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_063905-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b6c33d49b25b54551108e41cdb541c97fd700864 --- /dev/null +++ b/reward/wandb/run-20260530_063905-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T06:39:06.05490613+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpjsww3w3q/port-2412569.txt","pid":2412569,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T06:39:06.055506751+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2412569} +{"time":"2026-05-30T06:39:06.055494969+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2412569-2412719-1681422895/socket","Net":"unix"}} +{"time":"2026-05-30T06:39:06.2353733+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T06:39:06.244525054+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T06:39:06.644565256+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T06:39:07.148054071+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"n8q3wq6nxi3b"} +{"time":"2026-05-30T06:39:07.337938249+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"n8q3wq6nxi3b"} +{"time":"2026-05-30T06:39:07.34042734+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T06:39:07.341225439+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T06:39:07.341241498+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T06:39:07.341258579+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T06:39:07.341276174+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T06:39:07.341274108+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T06:39:07.34136357+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T06:39:07.34138241+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T06:39:07.341401134+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2412569-2412719-1681422895/socket","Net":"unix"}} +{"time":"2026-05-30T06:39:07.34144612+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_063905-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_063905-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3ea4fb511f0cc5fcd987482e587bef8c32f525fc --- /dev/null +++ b/reward/wandb/run-20260530_063905-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T06:39:06.244690631+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T06:39:06.642595389+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T06:39:06.642793447+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:06.642910518+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:06.644506607+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:06.644619048+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:06.644674388+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:06.648017091+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T06:39:06.940513948+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T06:39:07.147476004+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":10,"current":27} +{"time":"2026-05-30T06:39:07.283561627+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9c51a543ad6c1b227ffb5e8c3a13926cef4f8f0e528a3e85370b41607c5ba04a"} +{"time":"2026-05-30T06:39:07.28374234+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B42A60A405A11317e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063905-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9c51a543ad6c1b227ffb5e8c3a13926cef4f8f0e528a3e85370b41607c5ba04a, Size: 7996}"} +{"time":"2026-05-30T06:39:07.284056632+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f6d03412f4056c00a2264b367afc8b9766705c6a1e9b9f641dfbc7dba1086e0f"} +{"time":"2026-05-30T06:39:07.284128343+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c4a99f9998a12c3798a780d31dc151bfd8b79eef8c7f91d38b9b3c9c58f0fe72"} +{"time":"2026-05-30T06:39:07.284132103+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5000161ea1075e98310624ad182b5065740d7af7afd2cb8dc112375e983f36e6"} +{"time":"2026-05-30T06:39:07.284247627+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B42A60A385E98C29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063905-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c4a99f9998a12c3798a780d31dc151bfd8b79eef8c7f91d38b9b3c9c58f0fe72, Size: 2982}"} +{"time":"2026-05-30T06:39:07.284339004+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B42A60A385A10529678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063905-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5000161ea1075e98310624ad182b5065740d7af7afd2cb8dc112375e983f36e6, Size: 343}"} +{"time":"2026-05-30T06:39:07.284353812+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=197277e08c852de3d8f87882259b6178e5a3bdd1e8bd30967c340fdb89802018"} +{"time":"2026-05-30T06:39:07.284174193+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B42A60A464E0EF4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063905-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f6d03412f4056c00a2264b367afc8b9766705c6a1e9b9f641dfbc7dba1086e0f, Size: 2850}"} +{"time":"2026-05-30T06:39:07.284554128+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B42A60A422677F17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063905-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=197277e08c852de3d8f87882259b6178e5a3bdd1e8bd30967c340fdb89802018, Size: 111}"} +{"time":"2026-05-30T06:39:07.290872409+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=43cdfb4966d9902bb2657b32be8c3b776d26f51680b4425297ba292f938e5070"} +{"time":"2026-05-30T06:39:07.290992105+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B42A60A41AF7A617e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063905-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223907Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=43cdfb4966d9902bb2657b32be8c3b776d26f51680b4425297ba292f938e5070, Size: 14790}"} +{"time":"2026-05-30T06:39:07.291881333+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T06:39:07.334580507+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T06:39:07.340477119+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:07.340505075+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:07.340594381+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:07.34061829+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_063905-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_063905-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3cd2fa34378d925453c447605e496c018f0edaf6 --- /dev/null +++ b/reward/wandb/run-20260530_063905-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 06:39:05,989 INFO MainThread:2412569 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 06:39:05,989 INFO MainThread:2412569 [wandb_setup.py:_flush():81] Configure stats pid to 2412569 +2026-05-30 06:39:05,989 INFO MainThread:2412569 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 06:39:05,989 INFO MainThread:2412569 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063905-fmmkx1a5/logs/debug.log +2026-05-30 06:39:05,990 INFO MainThread:2412569 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063905-fmmkx1a5/logs/debug-internal.log +2026-05-30 06:39:05,990 INFO MainThread:2412569 [wandb_init.py:init():844] calling init triggers +2026-05-30 06:39:05,990 INFO MainThread:2412569 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 10, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 06:39:05,990 INFO MainThread:2412569 [wandb_init.py:init():892] starting backend +2026-05-30 06:39:06,235 INFO MainThread:2412569 [wandb_init.py:init():895] sending inform_init request +2026-05-30 06:39:06,240 INFO MainThread:2412569 [wandb_init.py:init():903] backend started and connected +2026-05-30 06:39:06,242 INFO MainThread:2412569 [wandb_init.py:init():973] updated telemetry +2026-05-30 06:39:06,246 INFO MainThread:2412569 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 06:39:06,934 INFO MainThread:2412569 [wandb_init.py:init():1037] run resumed +2026-05-30 06:39:06,938 INFO MainThread:2412569 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 06:39:07,028 INFO MainThread:2412569 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 06:39:07,028 INFO MainThread:2412569 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 06:39:07,028 INFO MainThread:2412569 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 06:39:07,028 INFO MainThread:2412569 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 06:39:07,032 INFO MainThread:2412569 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 06:39:07,147 INFO MainThread:2412569 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 06:39:07,147 INFO MainThread:2412569 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 06:39:07,147 INFO MainThread:2412569 [wandb_run.py:_restore():2472] restore +2026-05-30 06:39:07,147 INFO MainThread:2412569 [wandb_run.py:_restore():2478] restore done +2026-05-30 06:39:07,339 INFO MainThread:2412569 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_063905-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_063905-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4d15260092b6ae6f1e71b2108e33e184f7540879 Binary files /dev/null and b/reward/wandb/run-20260530_063905-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_063912-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_063912-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_063912-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_063912-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_063912-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34257318f8ff034f91e8b82931a8beba5e8c217c --- /dev/null +++ b/reward/wandb/run-20260530_063912-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + dbinq0zgbogrdo4ohjz5fcgj9q0fsxup: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=10 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625483104256" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-29T22:39:12.095450Z" + writerId: dbinq0zgbogrdo4ohjz5fcgj9q0fsxup + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 10 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_063912-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_063912-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5627c7ef5fc72c4661d3116ab264d566978f13fa --- /dev/null +++ b/reward/wandb/run-20260530_063912-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 10 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.5078 avg length: 125.81 diff --git a/reward/wandb/run-20260530_063912-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_063912-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_063912-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_063912-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_063912-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..16893d6f849ded3e21c2030c99fe39defc17272f --- /dev/null +++ b/reward/wandb/run-20260530_063912-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-29T22:39:12.095450Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=10" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625483104256" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "dbinq0zgbogrdo4ohjz5fcgj9q0fsxup" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_063912-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_063912-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e2eb660b93995848993e13f42634ee9227fec67a --- /dev/null +++ b/reward/wandb/run-20260530_063912-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":8003},"train/learning_rate":7.985792958513929e-07,"_runtime":8003,"reward/t_grpo_bonus":0.2718750000000003,"reward/avg_response_length":124.568359375,"reward/unbiased_bonus":0.09374999999999996,"_timestamp":1.7800760271916904e+09,"reward/train_step":1,"train/loss":-0.37430867552757263,"_step":26,"reward/acc":0.396484375} \ No newline at end of file diff --git a/reward/wandb/run-20260530_063912-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_063912-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..89c4f7fbb506626d5fd611a111175242f180dea1 --- /dev/null +++ b/reward/wandb/run-20260530_063912-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T06:39:12.162523739+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp7d0e5bp_/port-2412993.txt","pid":2412993,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T06:39:12.16338519+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2412993} +{"time":"2026-05-30T06:39:12.163371562+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2412993-2413150-1460272065/socket","Net":"unix"}} +{"time":"2026-05-30T06:39:12.343576579+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T06:39:12.353363274+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T06:39:12.660685832+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T06:39:13.087942334+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"1uaukwk5ablh"} +{"time":"2026-05-30T06:39:13.275060616+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"1uaukwk5ablh"} +{"time":"2026-05-30T06:39:13.275886909+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T06:39:13.276557466+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T06:39:13.27656763+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T06:39:13.276578158+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T06:39:13.276586555+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T06:39:13.27659938+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T06:39:13.27662021+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T06:39:13.276665646+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T06:39:13.276786795+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2412993-2413150-1460272065/socket","Net":"unix"}} +{"time":"2026-05-30T06:39:13.276847022+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_063912-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_063912-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d97ada41d18a5648f976fdeed31f13d4c7a136b9 --- /dev/null +++ b/reward/wandb/run-20260530_063912-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T06:39:12.35351855+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T06:39:12.6600652+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T06:39:12.660278429+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:12.660375555+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:12.660662843+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:12.660665328+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:12.660675667+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:12.666502544+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T06:39:12.875853852+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T06:39:13.087220351+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":10,"current":27} +{"time":"2026-05-30T06:39:13.219754876+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c0d7d2dadfcc432a72bfc8c3095ce4c7c52bbcc7a4e672fb22b43a3ea23be0cf"} +{"time":"2026-05-30T06:39:13.219919135+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B42A6205EAA24417e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063912-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c0d7d2dadfcc432a72bfc8c3095ce4c7c52bbcc7a4e672fb22b43a3ea23be0cf, Size: 343}"} +{"time":"2026-05-30T06:39:13.220152013+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f5813cacf454dc6af6bcda089428eef71ef1ba03f8d782182725a0258ac2e8cd"} +{"time":"2026-05-30T06:39:13.220276744+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B42A6205557F1729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063912-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f5813cacf454dc6af6bcda089428eef71ef1ba03f8d782182725a0258ac2e8cd, Size: 111}"} +{"time":"2026-05-30T06:39:13.2212563+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9938614b1c19350c0dcc6ddbf1ce1faafcb8ba27943beb810780499589e010b4"} +{"time":"2026-05-30T06:39:13.22138051+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B42A62055AEADF29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063912-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9938614b1c19350c0dcc6ddbf1ce1faafcb8ba27943beb810780499589e010b4, Size: 2850}"} +{"time":"2026-05-30T06:39:13.222225796+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d2e93fa343aa9817650da7cf4f6895b031cf2a6214bbbcf67f3bb9ba9809a37f"} +{"time":"2026-05-30T06:39:13.222347116+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B42A62064B4D024b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063912-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d2e93fa343aa9817650da7cf4f6895b031cf2a6214bbbcf67f3bb9ba9809a37f, Size: 2982}"} +{"time":"2026-05-30T06:39:13.222589666+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9ed5d42f4693d1171d03a43e8c42acfc8c96e014c77bd76a59eaddaf75801a6e"} +{"time":"2026-05-30T06:39:13.222684458+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B42A62057C113629678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063912-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9ed5d42f4693d1171d03a43e8c42acfc8c96e014c77bd76a59eaddaf75801a6e, Size: 7996}"} +{"time":"2026-05-30T06:39:13.228579646+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f6b029b007050ae5a94be05d550a604be30b9636363b4607152993d4ba6be534"} +{"time":"2026-05-30T06:39:13.228687612+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B42A6205F3898517e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063912-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T223913Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f6b029b007050ae5a94be05d550a604be30b9636363b4607152993d4ba6be534, Size: 14790}"} +{"time":"2026-05-30T06:39:13.229029313+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T06:39:13.270666789+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T06:39:13.275905429+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:13.275920252+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:13.275979262+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T06:39:13.27599205+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_063912-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_063912-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..48a6ebfc059619a5f59f5e083a56349224718d89 --- /dev/null +++ b/reward/wandb/run-20260530_063912-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 06:39:12,098 INFO MainThread:2412993 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 06:39:12,098 INFO MainThread:2412993 [wandb_setup.py:_flush():81] Configure stats pid to 2412993 +2026-05-30 06:39:12,098 INFO MainThread:2412993 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 06:39:12,098 INFO MainThread:2412993 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063912-fmmkx1a5/logs/debug.log +2026-05-30 06:39:12,098 INFO MainThread:2412993 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_063912-fmmkx1a5/logs/debug-internal.log +2026-05-30 06:39:12,098 INFO MainThread:2412993 [wandb_init.py:init():844] calling init triggers +2026-05-30 06:39:12,098 INFO MainThread:2412993 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 10, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 06:39:12,098 INFO MainThread:2412993 [wandb_init.py:init():892] starting backend +2026-05-30 06:39:12,344 INFO MainThread:2412993 [wandb_init.py:init():895] sending inform_init request +2026-05-30 06:39:12,349 INFO MainThread:2412993 [wandb_init.py:init():903] backend started and connected +2026-05-30 06:39:12,350 INFO MainThread:2412993 [wandb_init.py:init():973] updated telemetry +2026-05-30 06:39:12,353 INFO MainThread:2412993 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 06:39:12,869 INFO MainThread:2412993 [wandb_init.py:init():1037] run resumed +2026-05-30 06:39:12,873 INFO MainThread:2412993 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 06:39:12,964 INFO MainThread:2412993 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 06:39:12,964 INFO MainThread:2412993 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 06:39:12,964 INFO MainThread:2412993 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 06:39:12,964 INFO MainThread:2412993 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 06:39:12,967 INFO MainThread:2412993 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 06:39:13,087 INFO MainThread:2412993 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 06:39:13,087 INFO MainThread:2412993 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 06:39:13,087 INFO MainThread:2412993 [wandb_run.py:_restore():2472] restore +2026-05-30 06:39:13,087 INFO MainThread:2412993 [wandb_run.py:_restore():2478] restore done +2026-05-30 06:39:13,275 INFO MainThread:2412993 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_063912-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_063912-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8b1c21a4da0458264d064b6f1542104f78d7a7df Binary files /dev/null and b/reward/wandb/run-20260530_063912-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_094731-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_094731-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78722a5d41395696abb174b7dce72b29901deb5a --- /dev/null +++ b/reward/wandb/run-20260530_094731-fmmkx1a5/files/config.yaml @@ -0,0 +1,229 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + 079674nfrrgwsrat4zg8evfgzrmditvc: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=13 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + host: b1301b38c75f + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-30T01:47:31.852204Z" + writerId: 079674nfrrgwsrat4zg8evfgzrmditvc + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 13 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_094731-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_094731-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1bb31a6f6b51a28815e2d5e93ce49e0d92b9897c --- /dev/null +++ b/reward/wandb/run-20260530_094731-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,26 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-30T01:47:31.852204Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=13" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "writerId": "079674nfrrgwsrat4zg8evfgzrmditvc" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_094731-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_094731-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b6328dca9c6c35dacbb135c90e44b42db494d2b7 --- /dev/null +++ b/reward/wandb/run-20260530_094731-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":10662},"reward/unbiased_bonus":0.09374999999999996,"_timestamp":1.7801027754513717e+09,"train/loss":2.6498982906341553,"reward/t_grpo_bonus":0.2718750000000003,"train/learning_rate":8.236931423909135e-07,"reward/avg_response_length":124.568359375,"_step":28,"_runtime":10662,"reward/train_step":1,"reward/acc":0.396484375} \ No newline at end of file diff --git a/reward/wandb/run-20260530_094731-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_094731-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b875c47f990ab9539a9fa5c3a50da158a774420d --- /dev/null +++ b/reward/wandb/run-20260530_094731-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T09:47:31.916839156+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpt647xre4/port-2731469.txt","pid":2731469,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T09:47:31.91838095+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2731469} +{"time":"2026-05-30T09:47:31.918319682+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2731469-2731623-3039953975/socket","Net":"unix"}} +{"time":"2026-05-30T09:47:32.097564109+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T09:47:32.106616092+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T09:47:33.014223526+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T09:47:33.432787388+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"tha9p33dbfep"} +{"time":"2026-05-30T09:47:33.626946127+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"tha9p33dbfep"} +{"time":"2026-05-30T09:47:33.627587973+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T09:47:33.62814057+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T09:47:33.62815477+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T09:47:33.628170156+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T09:47:33.628185046+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T09:47:33.62818756+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T09:47:33.628270947+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2731469-2731623-3039953975/socket","Net":"unix"}} +{"time":"2026-05-30T09:47:33.628307197+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T09:47:33.628324419+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T09:47:33.628340851+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_094731-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_094731-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f5d2fb57e8aa894063bebb7ebc8d5abf25308f03 --- /dev/null +++ b/reward/wandb/run-20260530_094731-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T09:47:32.106836159+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T09:47:33.013682969+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T09:47:33.013897003+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T09:47:33.013983481+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T09:47:33.014205055+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T09:47:33.014247677+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T09:47:33.014245881+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T09:47:33.018915796+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T09:47:33.224918613+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T09:47:33.431972018+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":13,"current":29} +{"time":"2026-05-30T09:47:33.570787536+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c45fafa0542260256dca1005ecd8e0d2f82fae4c4b0341e5f1f290eb7931f118"} +{"time":"2026-05-30T09:47:33.570905525+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B434A9174486374b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_094731-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c45fafa0542260256dca1005ecd8e0d2f82fae4c4b0341e5f1f290eb7931f118, Size: 111}"} +{"time":"2026-05-30T09:47:33.571101573+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1f88bdb5e21f0295b019d5dbc158dc6787b9a3a2e015e47af55091e4b75b34fc"} +{"time":"2026-05-30T09:47:33.571203252+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B434A9174D04654b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_094731-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1f88bdb5e21f0295b019d5dbc158dc6787b9a3a2e015e47af55091e4b75b34fc, Size: 976}"} +{"time":"2026-05-30T09:47:33.571354598+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=afc03f752fc13b4f4833c92448558b78f573abd273dfaff983fd299a24ff8508"} +{"time":"2026-05-30T09:47:33.571564821+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B434A9165BB5EE29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_094731-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=afc03f752fc13b4f4833c92448558b78f573abd273dfaff983fd299a24ff8508, Size: 2982}"} +{"time":"2026-05-30T09:47:33.573102419+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7021dccf51e8dd9969d156a3deb0191d3892ed90cbf926b842bf56cba57a40e4"} +{"time":"2026-05-30T09:47:33.57313793+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d8fc626247b1ddaa115ad5be3b809756c47825d6645ce9417eb0ee239d65ffc0"} +{"time":"2026-05-30T09:47:33.573240201+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B434A91675441029678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_094731-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d8fc626247b1ddaa115ad5be3b809756c47825d6645ce9417eb0ee239d65ffc0, Size: 343}"} +{"time":"2026-05-30T09:47:33.573261508+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B434A916753E1C29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_094731-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7021dccf51e8dd9969d156a3deb0191d3892ed90cbf926b842bf56cba57a40e4, Size: 5678}"} +{"time":"2026-05-30T09:47:33.578161336+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=57802f662c4640efcdbe0d1b43316111519986e6b6065053707477247db9efbb"} +{"time":"2026-05-30T09:47:33.578213612+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B434A91708132A17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_094731-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T014733Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=57802f662c4640efcdbe0d1b43316111519986e6b6065053707477247db9efbb, Size: 14790}"} +{"time":"2026-05-30T09:47:33.578508188+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T09:47:33.622277936+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T09:47:33.627622937+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T09:47:33.627656793+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T09:47:33.627731189+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T09:47:33.627751113+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_094731-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_094731-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3b65b322c0af0aa05895b3f3c3c1270026e1f5b3 --- /dev/null +++ b/reward/wandb/run-20260530_094731-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 09:47:31,854 INFO MainThread:2731469 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 09:47:31,854 INFO MainThread:2731469 [wandb_setup.py:_flush():81] Configure stats pid to 2731469 +2026-05-30 09:47:31,854 INFO MainThread:2731469 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 09:47:31,854 INFO MainThread:2731469 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_094731-fmmkx1a5/logs/debug.log +2026-05-30 09:47:31,854 INFO MainThread:2731469 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_094731-fmmkx1a5/logs/debug-internal.log +2026-05-30 09:47:31,854 INFO MainThread:2731469 [wandb_init.py:init():844] calling init triggers +2026-05-30 09:47:31,854 INFO MainThread:2731469 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 13, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 09:47:31,854 INFO MainThread:2731469 [wandb_init.py:init():892] starting backend +2026-05-30 09:47:32,098 INFO MainThread:2731469 [wandb_init.py:init():895] sending inform_init request +2026-05-30 09:47:32,103 INFO MainThread:2731469 [wandb_init.py:init():903] backend started and connected +2026-05-30 09:47:32,104 INFO MainThread:2731469 [wandb_init.py:init():973] updated telemetry +2026-05-30 09:47:32,107 INFO MainThread:2731469 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 09:47:33,219 INFO MainThread:2731469 [wandb_init.py:init():1037] run resumed +2026-05-30 09:47:33,223 INFO MainThread:2731469 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 09:47:33,312 INFO MainThread:2731469 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 09:47:33,312 INFO MainThread:2731469 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 09:47:33,312 INFO MainThread:2731469 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 09:47:33,312 INFO MainThread:2731469 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 09:47:33,316 INFO MainThread:2731469 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 09:47:33,431 INFO MainThread:2731469 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 09:47:33,432 INFO MainThread:2731469 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 09:47:33,432 INFO MainThread:2731469 [wandb_run.py:_restore():2472] restore +2026-05-30 09:47:33,432 INFO MainThread:2731469 [wandb_run.py:_restore():2478] restore done +2026-05-30 09:47:33,627 INFO MainThread:2731469 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_094731-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_094731-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ce52f704d35d957efab49e922a2bc4afada2b9c4 Binary files /dev/null and b/reward/wandb/run-20260530_094731-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_140620-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_140620-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_140620-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_140620-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_140620-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b02e342306a45e26f178d853971599102d4b60d0 --- /dev/null +++ b/reward/wandb/run-20260530_140620-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + kdbqlc87pfa4827dfwy5t1uy0q9x8mlm: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=17 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625497964544" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-30T06:06:20.118971Z" + writerId: kdbqlc87pfa4827dfwy5t1uy0q9x8mlm + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 17 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_140620-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_140620-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f0d3a46254f2f1349ff3e089c9353153122f96a5 --- /dev/null +++ b/reward/wandb/run-20260530_140620-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 17 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.5703 avg length: 126.11 diff --git a/reward/wandb/run-20260530_140620-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_140620-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_140620-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_140620-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_140620-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c16a2a524ae4119adca5f360d9d349b4f6f1dbff --- /dev/null +++ b/reward/wandb/run-20260530_140620-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-30T06:06:20.118971Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=17" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625497964544" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "kdbqlc87pfa4827dfwy5t1uy0q9x8mlm" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_140620-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_140620-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9893747e33350ade48128c5980dfa82bc6763731 --- /dev/null +++ b/reward/wandb/run-20260530_140620-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/acc":0.396484375,"_wandb":{"runtime":14471},"_step":28,"_timestamp":1.7801027754513717e+09,"reward/train_step":1,"_runtime":14471,"reward/unbiased_bonus":0.09374999999999996,"train/loss":2.6498982906341553,"train/learning_rate":8.236931423909135e-07,"reward/avg_response_length":124.568359375,"reward/t_grpo_bonus":0.2718750000000003} \ No newline at end of file diff --git a/reward/wandb/run-20260530_140620-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_140620-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..41357e5855521f73c3778fc77794c6ee692f3ac2 --- /dev/null +++ b/reward/wandb/run-20260530_140620-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T14:06:20.183115304+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpi_3kt8h7/port-3190027.txt","pid":3190027,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T14:06:20.183655547+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3190027} +{"time":"2026-05-30T14:06:20.18361817+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3190027-3190190-2420569824/socket","Net":"unix"}} +{"time":"2026-05-30T14:06:20.364970105+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T14:06:20.375151007+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T14:06:20.77698137+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T14:06:21.187249055+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"70u9tf5hliry"} +{"time":"2026-05-30T14:06:21.368616282+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"70u9tf5hliry"} +{"time":"2026-05-30T14:06:21.369414395+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T14:06:21.370058221+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T14:06:21.370161068+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T14:06:21.370172608+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T14:06:21.370180635+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T14:06:21.370179882+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T14:06:21.370255072+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3190027-3190190-2420569824/socket","Net":"unix"}} +{"time":"2026-05-30T14:06:21.370271191+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T14:06:21.370283072+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T14:06:21.370296297+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_140620-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_140620-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a58193cbbda52101b7bbcb55e28fe1315dfd1cd5 --- /dev/null +++ b/reward/wandb/run-20260530_140620-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T14:06:20.375364145+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T14:06:20.774921564+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T14:06:20.77505547+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T14:06:20.775133526+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T14:06:20.776844104+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T14:06:20.776840356+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T14:06:20.776707255+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T14:06:20.778784289+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T14:06:20.976337875+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T14:06:21.18612904+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":17,"current":29} +{"time":"2026-05-30T14:06:21.312692131+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=940219a6702e987962031c95c5fa13e7fc51f4a4969bef409211702bfa966a04"} +{"time":"2026-05-30T14:06:21.31284697+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B442C86B7504BB29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_140620-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=940219a6702e987962031c95c5fa13e7fc51f4a4969bef409211702bfa966a04, Size: 343}"} +{"time":"2026-05-30T14:06:21.31287172+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f27f1c381d05ec21741a8e5867d5a8cce627d464715d3c85d657cb4b60a7e5c6"} +{"time":"2026-05-30T14:06:21.312980126+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B442C86C1FBA2417e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_140620-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f27f1c381d05ec21741a8e5867d5a8cce627d464715d3c85d657cb4b60a7e5c6, Size: 111}"} +{"time":"2026-05-30T14:06:21.313004426+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=df22f71663a4fc7619acae3ed07f861be3b0cf557401aeb2802b5d1cbf7e2414"} +{"time":"2026-05-30T14:06:21.312969606+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=62743c33ad115158ffb1ff2a75c30ac91e1d1d79256f188a817c477aeb9aefa4"} +{"time":"2026-05-30T14:06:21.315397101+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B442C86C86C94D4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_140620-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=df22f71663a4fc7619acae3ed07f861be3b0cf557401aeb2802b5d1cbf7e2414, Size: 7996}"} +{"time":"2026-05-30T14:06:21.315443962+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e5ff64c3aaef37539b19f715332c4e2bd796fb03aff90c41b425a68e4ee8226a"} +{"time":"2026-05-30T14:06:21.315694999+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B442C86C93594B4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_140620-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=62743c33ad115158ffb1ff2a75c30ac91e1d1d79256f188a817c477aeb9aefa4, Size: 2982}"} +{"time":"2026-05-30T14:06:21.316391023+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B442C86C3B11FB17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_140620-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e5ff64c3aaef37539b19f715332c4e2bd796fb03aff90c41b425a68e4ee8226a, Size: 2850}"} +{"time":"2026-05-30T14:06:21.324213423+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5b5164b833a327430e305ba590d77ac1e4071653079668bd52a75cebe86489f5"} +{"time":"2026-05-30T14:06:21.324354369+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B442C86B957FE429678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_140620-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T060621Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5b5164b833a327430e305ba590d77ac1e4071653079668bd52a75cebe86489f5, Size: 14790}"} +{"time":"2026-05-30T14:06:21.324658775+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T14:06:21.36428922+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T14:06:21.369454212+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T14:06:21.369484198+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T14:06:21.369572345+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T14:06:21.369593792+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_140620-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_140620-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..144ebd56985c3db15ff839c4657103fcf239023e --- /dev/null +++ b/reward/wandb/run-20260530_140620-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 14:06:20,121 INFO MainThread:3190027 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 14:06:20,121 INFO MainThread:3190027 [wandb_setup.py:_flush():81] Configure stats pid to 3190027 +2026-05-30 14:06:20,121 INFO MainThread:3190027 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 14:06:20,121 INFO MainThread:3190027 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_140620-fmmkx1a5/logs/debug.log +2026-05-30 14:06:20,121 INFO MainThread:3190027 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_140620-fmmkx1a5/logs/debug-internal.log +2026-05-30 14:06:20,121 INFO MainThread:3190027 [wandb_init.py:init():844] calling init triggers +2026-05-30 14:06:20,122 INFO MainThread:3190027 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 17, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 14:06:20,122 INFO MainThread:3190027 [wandb_init.py:init():892] starting backend +2026-05-30 14:06:20,365 INFO MainThread:3190027 [wandb_init.py:init():895] sending inform_init request +2026-05-30 14:06:20,371 INFO MainThread:3190027 [wandb_init.py:init():903] backend started and connected +2026-05-30 14:06:20,372 INFO MainThread:3190027 [wandb_init.py:init():973] updated telemetry +2026-05-30 14:06:20,377 INFO MainThread:3190027 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 14:06:20,970 INFO MainThread:3190027 [wandb_init.py:init():1037] run resumed +2026-05-30 14:06:20,974 INFO MainThread:3190027 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 14:06:21,063 INFO MainThread:3190027 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 14:06:21,063 INFO MainThread:3190027 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 14:06:21,063 INFO MainThread:3190027 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 14:06:21,063 INFO MainThread:3190027 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 14:06:21,067 INFO MainThread:3190027 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 14:06:21,186 INFO MainThread:3190027 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 14:06:21,186 INFO MainThread:3190027 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 14:06:21,186 INFO MainThread:3190027 [wandb_run.py:_restore():2472] restore +2026-05-30 14:06:21,186 INFO MainThread:3190027 [wandb_run.py:_restore():2478] restore done +2026-05-30 14:06:21,368 INFO MainThread:3190027 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_140620-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_140620-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..00277ffcc2bf54e24214e558814763ca1cc387f0 Binary files /dev/null and b/reward/wandb/run-20260530_140620-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_150423-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_150423-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_150423-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_150423-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_150423-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e5d7f580be1e1b759d03a6aeb308b380035bf81 --- /dev/null +++ b/reward/wandb/run-20260530_150423-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + 05mcyircheumu6jrpsckyym94f04yrik: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=18 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625581428736" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-30T07:04:23.209512Z" + writerId: 05mcyircheumu6jrpsckyym94f04yrik + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 18 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_150423-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_150423-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a6ef871d5aebab6a8e40c538bb7ac1ce2b52fdfe --- /dev/null +++ b/reward/wandb/run-20260530_150423-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 18 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.5273 avg length: 125.13 diff --git a/reward/wandb/run-20260530_150423-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_150423-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_150423-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_150423-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_150423-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1e31995bcf5f6f958c12a52c2f1c4e6b7df67e08 --- /dev/null +++ b/reward/wandb/run-20260530_150423-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-30T07:04:23.209512Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=18" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625581428736" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "05mcyircheumu6jrpsckyym94f04yrik" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_150423-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_150423-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1ecd1153688e8f5412c0fe27abd4ac27691fc2cc --- /dev/null +++ b/reward/wandb/run-20260530_150423-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/learning_rate":8.236931423909135e-07,"_wandb":{"runtime":15120},"reward/t_grpo_bonus":0.2718750000000003,"train/loss":2.6498982906341553,"reward/avg_response_length":124.568359375,"reward/unbiased_bonus":0.09374999999999996,"_timestamp":1.7801027754513717e+09,"reward/acc":0.396484375,"_runtime":15120,"_step":28,"reward/train_step":1} \ No newline at end of file diff --git a/reward/wandb/run-20260530_150423-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_150423-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..e14ddfed177614b982fb3eaa9f80d88d2bf94d35 --- /dev/null +++ b/reward/wandb/run-20260530_150423-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T15:04:23.274771691+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp3nry8ign/port-3293688.txt","pid":3293688,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T15:04:23.276949202+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3293688} +{"time":"2026-05-30T15:04:23.276945612+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3293688-3293853-1970419989/socket","Net":"unix"}} +{"time":"2026-05-30T15:04:23.455669766+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T15:04:23.465266038+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T15:04:23.77360682+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T15:04:24.199139944+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"foqhjreh5nnp"} +{"time":"2026-05-30T15:04:24.402908054+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"foqhjreh5nnp"} +{"time":"2026-05-30T15:04:24.403750716+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T15:04:24.404214611+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T15:04:24.404374597+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T15:04:24.404398928+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T15:04:24.404416813+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T15:04:24.404420724+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T15:04:24.404513463+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3293688-3293853-1970419989/socket","Net":"unix"}} +{"time":"2026-05-30T15:04:24.4045579+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T15:04:24.40458041+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T15:04:24.40460437+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_150423-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_150423-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ae9fc6d51147587b7a734a42da003faa92e953c9 --- /dev/null +++ b/reward/wandb/run-20260530_150423-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T15:04:23.465417184+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T15:04:23.773059387+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T15:04:23.773242647+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T15:04:23.773345464+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T15:04:23.77358277+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T15:04:23.773622591+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T15:04:23.773633336+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T15:04:23.778872793+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T15:04:23.986625206+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T15:04:24.198428175+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":18,"current":29} +{"time":"2026-05-30T15:04:24.33148027+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=68ef99708ed0241e9ad1291b9a4863800dc13d1b6fe4c57d334844af9a5bc6f9"} +{"time":"2026-05-30T15:04:24.331613457+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B445F3609865004b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_150423-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=68ef99708ed0241e9ad1291b9a4863800dc13d1b6fe4c57d334844af9a5bc6f9, Size: 343}"} +{"time":"2026-05-30T15:04:24.332362241+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9313386c196f7f2381e002cfea2b628ebb94c07c7194b866f9dac5f21fcaa4d8"} +{"time":"2026-05-30T15:04:24.33247146+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B445F3609FDBAB4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_150423-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9313386c196f7f2381e002cfea2b628ebb94c07c7194b866f9dac5f21fcaa4d8, Size: 2850}"} +{"time":"2026-05-30T15:04:24.332924049+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=da045e48f76e9f89ebd3bb46a4998a3210c0590e1ef1695826cc669223b5b9ef"} +{"time":"2026-05-30T15:04:24.33302992+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B445F3603F99D217e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_150423-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=da045e48f76e9f89ebd3bb46a4998a3210c0590e1ef1695826cc669223b5b9ef, Size: 7996}"} +{"time":"2026-05-30T15:04:24.334185247+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cf77824f2f39d58d5205e817ae5bebd8eb09d58a8aa08272127f03a5d4ab5fb0"} +{"time":"2026-05-30T15:04:24.334271286+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B445F35F98441A29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_150423-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cf77824f2f39d58d5205e817ae5bebd8eb09d58a8aa08272127f03a5d4ab5fb0, Size: 111}"} +{"time":"2026-05-30T15:04:24.334410223+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=496320bf137369893e2ad010f6a11510f0ee4fd9ff3517cfb8033569363330fa"} +{"time":"2026-05-30T15:04:24.334508344+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B445F360BC15D64b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_150423-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=496320bf137369893e2ad010f6a11510f0ee4fd9ff3517cfb8033569363330fa, Size: 2982}"} +{"time":"2026-05-30T15:04:24.342985014+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=36167da83d17fb4c4142d17409674898fbe229f414b68adda4f3b0b6667cd522"} +{"time":"2026-05-30T15:04:24.34315608+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B445F36035BE2017e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_150423-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T070424Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=36167da83d17fb4c4142d17409674898fbe229f414b68adda4f3b0b6667cd522, Size: 14790}"} +{"time":"2026-05-30T15:04:24.343697916+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T15:04:24.399989878+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T15:04:24.403769838+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T15:04:24.403784202+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T15:04:24.403828201+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T15:04:24.40384029+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_150423-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_150423-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e626799caa7184b682fc607d828397cf5c3fb502 --- /dev/null +++ b/reward/wandb/run-20260530_150423-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 15:04:23,212 INFO MainThread:3293688 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 15:04:23,212 INFO MainThread:3293688 [wandb_setup.py:_flush():81] Configure stats pid to 3293688 +2026-05-30 15:04:23,212 INFO MainThread:3293688 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 15:04:23,212 INFO MainThread:3293688 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_150423-fmmkx1a5/logs/debug.log +2026-05-30 15:04:23,212 INFO MainThread:3293688 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_150423-fmmkx1a5/logs/debug-internal.log +2026-05-30 15:04:23,212 INFO MainThread:3293688 [wandb_init.py:init():844] calling init triggers +2026-05-30 15:04:23,212 INFO MainThread:3293688 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 18, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 15:04:23,212 INFO MainThread:3293688 [wandb_init.py:init():892] starting backend +2026-05-30 15:04:23,456 INFO MainThread:3293688 [wandb_init.py:init():895] sending inform_init request +2026-05-30 15:04:23,461 INFO MainThread:3293688 [wandb_init.py:init():903] backend started and connected +2026-05-30 15:04:23,462 INFO MainThread:3293688 [wandb_init.py:init():973] updated telemetry +2026-05-30 15:04:23,465 INFO MainThread:3293688 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 15:04:23,980 INFO MainThread:3293688 [wandb_init.py:init():1037] run resumed +2026-05-30 15:04:23,985 INFO MainThread:3293688 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 15:04:24,073 INFO MainThread:3293688 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 15:04:24,074 INFO MainThread:3293688 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 15:04:24,074 INFO MainThread:3293688 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 15:04:24,074 INFO MainThread:3293688 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 15:04:24,077 INFO MainThread:3293688 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 15:04:24,198 INFO MainThread:3293688 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 15:04:24,198 INFO MainThread:3293688 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 15:04:24,198 INFO MainThread:3293688 [wandb_run.py:_restore():2472] restore +2026-05-30 15:04:24,199 INFO MainThread:3293688 [wandb_run.py:_restore():2478] restore done +2026-05-30 15:04:24,403 INFO MainThread:3293688 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_150423-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_150423-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8e21fb5b1c5c2a225121f6287834af6037c540b9 Binary files /dev/null and b/reward/wandb/run-20260530_150423-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_161011-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_161011-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_161011-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_161011-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_161011-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3902444db4922f7433dbf1dd1f2afb5af576ef34 --- /dev/null +++ b/reward/wandb/run-20260530_161011-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + fjbxu39z563nkzzw9ofi3oioolmnxoke: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=19 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625587679232" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-30T08:10:11.890374Z" + writerId: fjbxu39z563nkzzw9ofi3oioolmnxoke + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 19 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_161011-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_161011-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..40bd356456a546faa7d49679ebe550cbecc8ba72 --- /dev/null +++ b/reward/wandb/run-20260530_161011-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 19 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.4551 avg length: 125.24 diff --git a/reward/wandb/run-20260530_161011-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_161011-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_161011-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_161011-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_161011-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0327c8ae6cda694a172f18bd862a7ae38a43e4a3 --- /dev/null +++ b/reward/wandb/run-20260530_161011-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-30T08:10:11.890374Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=19" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625587679232" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "fjbxu39z563nkzzw9ofi3oioolmnxoke" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_161011-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_161011-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4f24a0b76fb196fa973d18f3c168e2a552faa58d --- /dev/null +++ b/reward/wandb/run-20260530_161011-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/acc":0.396484375,"_timestamp":1.7801027754513717e+09,"train/loss":2.6498982906341553,"reward/train_step":1,"train/learning_rate":8.236931423909135e-07,"_step":28,"_wandb":{"runtime":16125},"reward/t_grpo_bonus":0.2718750000000003,"reward/avg_response_length":124.568359375,"reward/unbiased_bonus":0.09374999999999996,"_runtime":16125} \ No newline at end of file diff --git a/reward/wandb/run-20260530_161011-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_161011-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..83bb1beddf03a1f304b696c46160052360a68113 --- /dev/null +++ b/reward/wandb/run-20260530_161011-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T16:10:11.956626028+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp8hj79q9v/port-3412798.txt","pid":3412798,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T16:10:11.957256552+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3412798-3412954-3466209454/socket","Net":"unix"}} +{"time":"2026-05-30T16:10:11.957345167+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3412798} +{"time":"2026-05-30T16:10:12.137181634+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T16:10:12.146100049+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T16:10:12.454723434+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T16:10:12.930311706+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"tsgd9dooxdpf"} +{"time":"2026-05-30T16:10:13.113683088+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"tsgd9dooxdpf"} +{"time":"2026-05-30T16:10:13.114989657+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T16:10:13.115590028+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T16:10:13.115597125+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T16:10:13.115606833+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T16:10:13.11561588+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T16:10:13.115627284+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T16:10:13.115673841+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3412798-3412954-3466209454/socket","Net":"unix"}} +{"time":"2026-05-30T16:10:13.115774181+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T16:10:13.115791661+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T16:10:13.11581104+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_161011-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_161011-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7bfe4b72b2c8272a306d061a5a43fe4d02e8c3ec --- /dev/null +++ b/reward/wandb/run-20260530_161011-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T16:10:12.146329446+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T16:10:12.454231668+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T16:10:12.454423989+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T16:10:12.454499163+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T16:10:12.454705731+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T16:10:12.454738166+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T16:10:12.454744269+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T16:10:12.459221291+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T16:10:12.715556275+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T16:10:12.929354076+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":19,"current":29} +{"time":"2026-05-30T16:10:13.058809856+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9b8f87a076ca4fe317265430e7f0107a45e1d9cc97f6176153941d3f69eed4c9"} +{"time":"2026-05-30T16:10:13.058948862+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B4498AC43C275129678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_161011-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9b8f87a076ca4fe317265430e7f0107a45e1d9cc97f6176153941d3f69eed4c9, Size: 343}"} +{"time":"2026-05-30T16:10:13.059215892+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5760df7e426fc1dd2c39816e965bbb648bca8299dfa381f40d93deedf5b0ee54"} +{"time":"2026-05-30T16:10:13.059350808+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B4498AC2FB23FC4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_161011-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5760df7e426fc1dd2c39816e965bbb648bca8299dfa381f40d93deedf5b0ee54, Size: 2982}"} +{"time":"2026-05-30T16:10:13.060716619+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6bda022a48d1915e987950a11e64800064fcf603c206e1ed87c9b1df48e5620b"} +{"time":"2026-05-30T16:10:13.060856162+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B4498AC299F72E17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_161011-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6bda022a48d1915e987950a11e64800064fcf603c206e1ed87c9b1df48e5620b, Size: 7996}"} +{"time":"2026-05-30T16:10:13.063976743+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1c7792d18817343b61f86276b4bf4210bdaa762e4d256946bf557df045dc6013"} +{"time":"2026-05-30T16:10:13.06407764+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B4498AC3294F0E4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_161011-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1c7792d18817343b61f86276b4bf4210bdaa762e4d256946bf557df045dc6013, Size: 2850}"} +{"time":"2026-05-30T16:10:13.069183574+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=754f0cd40404fe2aa4b004a5561232db3010c6b2f0d1eb90f5a8160b25d38667"} +{"time":"2026-05-30T16:10:13.06924842+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B4498AC46462F329678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_161011-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=754f0cd40404fe2aa4b004a5561232db3010c6b2f0d1eb90f5a8160b25d38667, Size: 14790}"} +{"time":"2026-05-30T16:10:13.069264962+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1704f56f82553988836618b8b821cd3a04991f2f3b243bd04453b25882ee39db"} +{"time":"2026-05-30T16:10:13.069376613+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B4498AC478BA5D29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_161011-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T081013Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1704f56f82553988836618b8b821cd3a04991f2f3b243bd04453b25882ee39db, Size: 111}"} +{"time":"2026-05-30T16:10:13.070287353+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T16:10:13.110519607+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T16:10:13.115007479+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T16:10:13.115019865+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T16:10:13.115067725+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T16:10:13.115079359+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_161011-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_161011-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3b4119fe768f00ac27661e50880d88a406b7a6f5 --- /dev/null +++ b/reward/wandb/run-20260530_161011-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 16:10:11,892 INFO MainThread:3412798 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 16:10:11,893 INFO MainThread:3412798 [wandb_setup.py:_flush():81] Configure stats pid to 3412798 +2026-05-30 16:10:11,893 INFO MainThread:3412798 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 16:10:11,893 INFO MainThread:3412798 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_161011-fmmkx1a5/logs/debug.log +2026-05-30 16:10:11,893 INFO MainThread:3412798 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_161011-fmmkx1a5/logs/debug-internal.log +2026-05-30 16:10:11,893 INFO MainThread:3412798 [wandb_init.py:init():844] calling init triggers +2026-05-30 16:10:11,893 INFO MainThread:3412798 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 19, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 16:10:11,893 INFO MainThread:3412798 [wandb_init.py:init():892] starting backend +2026-05-30 16:10:12,137 INFO MainThread:3412798 [wandb_init.py:init():895] sending inform_init request +2026-05-30 16:10:12,142 INFO MainThread:3412798 [wandb_init.py:init():903] backend started and connected +2026-05-30 16:10:12,144 INFO MainThread:3412798 [wandb_init.py:init():973] updated telemetry +2026-05-30 16:10:12,147 INFO MainThread:3412798 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 16:10:12,710 INFO MainThread:3412798 [wandb_init.py:init():1037] run resumed +2026-05-30 16:10:12,714 INFO MainThread:3412798 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 16:10:12,802 INFO MainThread:3412798 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 16:10:12,802 INFO MainThread:3412798 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 16:10:12,802 INFO MainThread:3412798 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 16:10:12,802 INFO MainThread:3412798 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 16:10:12,806 INFO MainThread:3412798 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 16:10:12,929 INFO MainThread:3412798 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 16:10:12,929 INFO MainThread:3412798 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 16:10:12,929 INFO MainThread:3412798 [wandb_run.py:_restore():2472] restore +2026-05-30 16:10:12,929 INFO MainThread:3412798 [wandb_run.py:_restore():2478] restore done +2026-05-30 16:10:13,113 INFO MainThread:3412798 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_161011-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_161011-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9adcfa2fb709e72ae4e5b40e629ecf4d6047c619 Binary files /dev/null and b/reward/wandb/run-20260530_161011-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_181659-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_181659-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_181659-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_181659-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_181659-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb23e69f3b388b3d59125f83d004cacc905670b3 --- /dev/null +++ b/reward/wandb/run-20260530_181659-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + vcnnf6xuys2qyf9ujxlym6utljz0e64j: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=21 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625595760640" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-30T10:16:59.061423Z" + writerId: vcnnf6xuys2qyf9ujxlym6utljz0e64j + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 21 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_181659-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_181659-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d0df4dbdf72bbf4a2a31c4cd6caa16d48cd9392e --- /dev/null +++ b/reward/wandb/run-20260530_181659-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 21 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.3711 avg length: 126.30 diff --git a/reward/wandb/run-20260530_181659-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_181659-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_181659-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_181659-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_181659-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4e3df5020d092c75acad468f79e0de2cef62c77d --- /dev/null +++ b/reward/wandb/run-20260530_181659-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-30T10:16:59.061423Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=21" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625595760640" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "vcnnf6xuys2qyf9ujxlym6utljz0e64j" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_181659-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_181659-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ed4b1fe4baeda14088165796fa611baa5797415e --- /dev/null +++ b/reward/wandb/run-20260530_181659-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/loss":-0.3729729354381562,"reward/train_step":1,"reward/acc":0.396484375,"reward/avg_response_length":124.568359375,"_step":38,"_wandb":{"runtime":18034},"reward/unbiased_bonus":0.09374999999999996,"train/learning_rate":8.997213817017502e-07,"_runtime":18034,"reward/t_grpo_bonus":0.2718750000000003,"_timestamp":1.7801299761845806e+09} \ No newline at end of file diff --git a/reward/wandb/run-20260530_181659-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_181659-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..9952b763a265f3701646f8c59880d9bfa8f91677 --- /dev/null +++ b/reward/wandb/run-20260530_181659-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T18:16:59.125954785+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpjzp9n1_1/port-3654974.txt","pid":3654974,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T18:16:59.126545534+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3654974} +{"time":"2026-05-30T18:16:59.126493414+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3654974-3655137-2066590616/socket","Net":"unix"}} +{"time":"2026-05-30T18:16:59.307168387+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T18:16:59.316768208+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T18:16:59.619042785+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T18:17:00.101150025+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"yzfikbqx9htw"} +{"time":"2026-05-30T18:17:00.289716114+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"yzfikbqx9htw"} +{"time":"2026-05-30T18:17:00.290516376+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T18:17:00.291235611+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T18:17:00.291301447+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T18:17:00.291345558+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T18:17:00.291367226+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T18:17:00.291366491+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T18:17:00.291472928+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3654974-3655137-2066590616/socket","Net":"unix"}} +{"time":"2026-05-30T18:17:00.291503699+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T18:17:00.291535212+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T18:17:00.291567589+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_181659-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_181659-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5733e630324d19a2606aae5d95a6cb75f8dbd0e7 --- /dev/null +++ b/reward/wandb/run-20260530_181659-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T18:16:59.316898071+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T18:16:59.6167338+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T18:16:59.616927891+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T18:16:59.617004963+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T18:16:59.617226498+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T18:16:59.617280238+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T18:16:59.617281053+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T18:16:59.622270239+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T18:16:59.880653691+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T18:17:00.100187581+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":21,"current":39} +{"time":"2026-05-30T18:17:00.239223005+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9c77148662bdcb6a5321acc43eeab44574e07c938bc8cab69ecfccd3201bdc90"} +{"time":"2026-05-30T18:17:00.239389658+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B45075F22971344b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_181659-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9c77148662bdcb6a5321acc43eeab44574e07c938bc8cab69ecfccd3201bdc90, Size: 2982}"} +{"time":"2026-05-30T18:17:00.239773503+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=270c164144f28fd7f50d032a56bc6597aca8005c11831063b6dfe39dfb7de5ec"} +{"time":"2026-05-30T18:17:00.239902357+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B45075F224CB374b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_181659-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=270c164144f28fd7f50d032a56bc6597aca8005c11831063b6dfe39dfb7de5ec, Size: 111}"} +{"time":"2026-05-30T18:17:00.240601382+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce176da73ce44e2bd926b62c1ebfed65026fbf9b7ddbc7917fa97820c02f8bf4"} +{"time":"2026-05-30T18:17:00.240715369+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7579e83fb2646ee7cc21f7cae0ad2ca781d33b8bc2ad14567075bb5d26af7cd9"} +{"time":"2026-05-30T18:17:00.240736527+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B45075F1B8689417e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_181659-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce176da73ce44e2bd926b62c1ebfed65026fbf9b7ddbc7917fa97820c02f8bf4, Size: 7996}"} +{"time":"2026-05-30T18:17:00.240822094+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B45075F367E1EC29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_181659-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7579e83fb2646ee7cc21f7cae0ad2ca781d33b8bc2ad14567075bb5d26af7cd9, Size: 2850}"} +{"time":"2026-05-30T18:17:00.243731827+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=da8063d5b5042e932c1964835059b1dca55d879272e6e2a499fd3d5c7af1f194"} +{"time":"2026-05-30T18:17:00.243900368+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B45075F1CA27F217e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_181659-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=da8063d5b5042e932c1964835059b1dca55d879272e6e2a499fd3d5c7af1f194, Size: 344}"} +{"time":"2026-05-30T18:17:00.244259653+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=140f3efbee842533bc1d2309a5b53c6e452a8a458cef59e07a37722ffed4e69a"} +{"time":"2026-05-30T18:17:00.24434367+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B45075F3523DC929678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_181659-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T101700Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=140f3efbee842533bc1d2309a5b53c6e452a8a458cef59e07a37722ffed4e69a, Size: 14790}"} +{"time":"2026-05-30T18:17:00.245228319+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T18:17:00.286455892+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T18:17:00.290529423+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T18:17:00.290546012+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T18:17:00.290600232+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T18:17:00.290614907+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_181659-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_181659-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3b490a2f94fffd210236677e8b5fce8c8c465f40 --- /dev/null +++ b/reward/wandb/run-20260530_181659-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 18:16:59,063 INFO MainThread:3654974 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 18:16:59,063 INFO MainThread:3654974 [wandb_setup.py:_flush():81] Configure stats pid to 3654974 +2026-05-30 18:16:59,063 INFO MainThread:3654974 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 18:16:59,063 INFO MainThread:3654974 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_181659-fmmkx1a5/logs/debug.log +2026-05-30 18:16:59,063 INFO MainThread:3654974 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_181659-fmmkx1a5/logs/debug-internal.log +2026-05-30 18:16:59,063 INFO MainThread:3654974 [wandb_init.py:init():844] calling init triggers +2026-05-30 18:16:59,063 INFO MainThread:3654974 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 21, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 18:16:59,063 INFO MainThread:3654974 [wandb_init.py:init():892] starting backend +2026-05-30 18:16:59,307 INFO MainThread:3654974 [wandb_init.py:init():895] sending inform_init request +2026-05-30 18:16:59,312 INFO MainThread:3654974 [wandb_init.py:init():903] backend started and connected +2026-05-30 18:16:59,314 INFO MainThread:3654974 [wandb_init.py:init():973] updated telemetry +2026-05-30 18:16:59,317 INFO MainThread:3654974 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 18:16:59,874 INFO MainThread:3654974 [wandb_init.py:init():1037] run resumed +2026-05-30 18:16:59,878 INFO MainThread:3654974 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 18:16:59,968 INFO MainThread:3654974 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 18:16:59,968 INFO MainThread:3654974 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 18:16:59,968 INFO MainThread:3654974 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 18:16:59,968 INFO MainThread:3654974 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 18:16:59,971 INFO MainThread:3654974 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 18:17:00,100 INFO MainThread:3654974 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 18:17:00,100 INFO MainThread:3654974 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 18:17:00,100 INFO MainThread:3654974 [wandb_run.py:_restore():2472] restore +2026-05-30 18:17:00,100 INFO MainThread:3654974 [wandb_run.py:_restore():2478] restore done +2026-05-30 18:17:00,289 INFO MainThread:3654974 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_181659-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_181659-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b81ff1422c0d7d335385c063afffaa1380205e42 Binary files /dev/null and b/reward/wandb/run-20260530_181659-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_192830-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_192830-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_192830-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_192830-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_192830-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b4c8277b5fb69a741b47db8d78fd4d6d7e77621 --- /dev/null +++ b/reward/wandb/run-20260530_192830-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + mrcwo0zqfrebu00tmykgciyxn3r0lssp: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=22 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625595719680" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-30T11:28:30.148048Z" + writerId: mrcwo0zqfrebu00tmykgciyxn3r0lssp + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 22 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_192830-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_192830-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0ec00251ec2a559b5b3a15e262e4a20af3ed5559 --- /dev/null +++ b/reward/wandb/run-20260530_192830-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 22 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.4434 avg length: 126.28 diff --git a/reward/wandb/run-20260530_192830-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_192830-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_192830-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_192830-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_192830-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7d792265f6bf874ddd7f5bd1a2eabc7bfe7a3766 --- /dev/null +++ b/reward/wandb/run-20260530_192830-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-30T11:28:30.148048Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=22" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625595719680" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "mrcwo0zqfrebu00tmykgciyxn3r0lssp" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_192830-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_192830-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fde7b5d3709640b04c3c6b8b73ce9fd77cceb7a7 --- /dev/null +++ b/reward/wandb/run-20260530_192830-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/unbiased_bonus":0.09374999999999996,"reward/t_grpo_bonus":0.2718750000000003,"_step":38,"reward/acc":0.396484375,"reward/avg_response_length":124.568359375,"_wandb":{"runtime":19367},"_timestamp":1.7801299761845806e+09,"reward/train_step":1,"train/loss":-0.3729729354381562,"train/learning_rate":8.997213817017502e-07,"_runtime":19367} \ No newline at end of file diff --git a/reward/wandb/run-20260530_192830-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_192830-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..0a8d15a2e000edf6079744a223686c761641555b --- /dev/null +++ b/reward/wandb/run-20260530_192830-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T19:28:30.212321462+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpbrxzafyd/port-3747859.txt","pid":3747859,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T19:28:30.212910161+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3747859} +{"time":"2026-05-30T19:28:30.212866705+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3747859-3747953-2930111190/socket","Net":"unix"}} +{"time":"2026-05-30T19:28:30.39373951+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T19:28:30.404404309+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T19:28:30.78678418+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T19:28:31.224750508+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"l304ij8blnea"} +{"time":"2026-05-30T19:28:31.406183889+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"l304ij8blnea"} +{"time":"2026-05-30T19:28:31.407143737+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T19:28:31.407662281+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T19:28:31.408176687+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T19:28:31.408221682+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T19:28:31.408244536+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T19:28:31.408239151+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T19:28:31.408314768+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T19:28:31.408328503+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T19:28:31.408395308+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3747859-3747953-2930111190/socket","Net":"unix"}} +{"time":"2026-05-30T19:28:31.408444319+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_192830-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_192830-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..48ac0d12781d177ba6e11a0be2ae000b840dead8 --- /dev/null +++ b/reward/wandb/run-20260530_192830-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T19:28:30.404588488+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T19:28:30.784727544+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T19:28:30.784914+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T19:28:30.78499726+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T19:28:30.78557804+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T19:28:30.786560605+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T19:28:30.786729419+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T19:28:30.790200746+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T19:28:30.996776566+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T19:28:31.223802352+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":22,"current":39} +{"time":"2026-05-30T19:28:31.354569234+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=371fc07105a5c4b4438239ac7db8162951f5d357397073762264abf60aa929e3"} +{"time":"2026-05-30T19:28:31.354700003+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B4545D0C6EA54B4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_192830-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=371fc07105a5c4b4438239ac7db8162951f5d357397073762264abf60aa929e3, Size: 7996}"} +{"time":"2026-05-30T19:28:31.355649203+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4eeb44f194f3dc14529fbfc0db97e9029252a1d2308e11a2da190cc34c460f92"} +{"time":"2026-05-30T19:28:31.355760795+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B4545D0BFFB74517e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_192830-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4eeb44f194f3dc14529fbfc0db97e9029252a1d2308e11a2da190cc34c460f92, Size: 2982}"} +{"time":"2026-05-30T19:28:31.355882301+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a5dda5f2186afddd194913f628389f3b10b9deb61b66a68de8c0cee7d0289dfe"} +{"time":"2026-05-30T19:28:31.356033607+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B4545D0DAD3E8C29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_192830-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a5dda5f2186afddd194913f628389f3b10b9deb61b66a68de8c0cee7d0289dfe, Size: 344}"} +{"time":"2026-05-30T19:28:31.356074665+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3d1d90766f72f52842932774bb0852ef11518eb05f46fe7208ebf8a22ca5ef70"} +{"time":"2026-05-30T19:28:31.356228933+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B4545D0C82418B4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_192830-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3d1d90766f72f52842932774bb0852ef11518eb05f46fe7208ebf8a22ca5ef70, Size: 111}"} +{"time":"2026-05-30T19:28:31.360082199+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=62cd77718685ee0bb3c5a6a005836f3b30664f0b76f01a161cd7bbb906439bb6"} +{"time":"2026-05-30T19:28:31.360263383+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B4545D0CABF0674b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_192830-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=62cd77718685ee0bb3c5a6a005836f3b30664f0b76f01a161cd7bbb906439bb6, Size: 2850}"} +{"time":"2026-05-30T19:28:31.360569464+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2654a4f0820f637e022df70e2b69d937e38094f9be7341f2ce27931782c958a3"} +{"time":"2026-05-30T19:28:31.360689037+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B4545D0C0018ED17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_192830-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T112831Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2654a4f0820f637e022df70e2b69d937e38094f9be7341f2ce27931782c958a3, Size: 14790}"} +{"time":"2026-05-30T19:28:31.361236227+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T19:28:31.402120524+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T19:28:31.407159017+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T19:28:31.407176995+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T19:28:31.40724706+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T19:28:31.407263995+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_192830-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_192830-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..398109305fd44f2dba057f7603e5e4770a5859cf --- /dev/null +++ b/reward/wandb/run-20260530_192830-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 19:28:30,150 INFO MainThread:3747859 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 19:28:30,150 INFO MainThread:3747859 [wandb_setup.py:_flush():81] Configure stats pid to 3747859 +2026-05-30 19:28:30,150 INFO MainThread:3747859 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 19:28:30,150 INFO MainThread:3747859 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_192830-fmmkx1a5/logs/debug.log +2026-05-30 19:28:30,150 INFO MainThread:3747859 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_192830-fmmkx1a5/logs/debug-internal.log +2026-05-30 19:28:30,150 INFO MainThread:3747859 [wandb_init.py:init():844] calling init triggers +2026-05-30 19:28:30,150 INFO MainThread:3747859 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 22, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 19:28:30,150 INFO MainThread:3747859 [wandb_init.py:init():892] starting backend +2026-05-30 19:28:30,394 INFO MainThread:3747859 [wandb_init.py:init():895] sending inform_init request +2026-05-30 19:28:30,400 INFO MainThread:3747859 [wandb_init.py:init():903] backend started and connected +2026-05-30 19:28:30,401 INFO MainThread:3747859 [wandb_init.py:init():973] updated telemetry +2026-05-30 19:28:30,406 INFO MainThread:3747859 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 19:28:30,991 INFO MainThread:3747859 [wandb_init.py:init():1037] run resumed +2026-05-30 19:28:30,995 INFO MainThread:3747859 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 19:28:31,084 INFO MainThread:3747859 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 19:28:31,084 INFO MainThread:3747859 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 19:28:31,084 INFO MainThread:3747859 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 19:28:31,084 INFO MainThread:3747859 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 19:28:31,087 INFO MainThread:3747859 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 19:28:31,223 INFO MainThread:3747859 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 19:28:31,224 INFO MainThread:3747859 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 19:28:31,224 INFO MainThread:3747859 [wandb_run.py:_restore():2472] restore +2026-05-30 19:28:31,224 INFO MainThread:3747859 [wandb_run.py:_restore():2478] restore done +2026-05-30 19:28:31,406 INFO MainThread:3747859 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_192830-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_192830-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8ffc766ad40b86ce0c54960ac1e4e8363c4828c0 Binary files /dev/null and b/reward/wandb/run-20260530_192830-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_212924-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_212924-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_212924-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_212924-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_212924-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be8b083c281d374460e4658c6968eab1af60cb38 --- /dev/null +++ b/reward/wandb/run-20260530_212924-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + 1qjs3tagx2ytglm665nd9a1d7ywq78e9: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=24 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625597681664" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-30T13:29:24.943978Z" + writerId: 1qjs3tagx2ytglm665nd9a1d7ywq78e9 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 24 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_212924-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_212924-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e756902c1b3bd86f2835a53ac5309a3d662efac9 --- /dev/null +++ b/reward/wandb/run-20260530_212924-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 24 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.4551 avg length: 124.63 diff --git a/reward/wandb/run-20260530_212924-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_212924-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_212924-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_212924-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260530_212924-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bf6b22f205f7d4b2b1d5390185661da3f542e76f --- /dev/null +++ b/reward/wandb/run-20260530_212924-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-30T13:29:24.943978Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=24" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625597681664" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "1qjs3tagx2ytglm665nd9a1d7ywq78e9" +} \ No newline at end of file diff --git a/reward/wandb/run-20260530_212924-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_212924-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5f2681d70f42068b4e7aa352dbbdd5020cc09f61 --- /dev/null +++ b/reward/wandb/run-20260530_212924-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":20949,"reward/unbiased_bonus":0.09374999999999996,"reward/train_step":1,"reward/t_grpo_bonus":0.2718750000000003,"_timestamp":1.7801299761845806e+09,"train/learning_rate":8.997213817017502e-07,"reward/avg_response_length":124.568359375,"_step":38,"train/loss":-0.3729729354381562,"reward/acc":0.396484375,"_wandb":{"runtime":20949}} \ No newline at end of file diff --git a/reward/wandb/run-20260530_212924-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260530_212924-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..0a9ec32af9da67950034a20c5b52ae62afacde6b --- /dev/null +++ b/reward/wandb/run-20260530_212924-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-30T21:29:25.011122855+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmphqa80_y0/port-3842490.txt","pid":3842490,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-30T21:29:25.012117078+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3842490} +{"time":"2026-05-30T21:29:25.012092403+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3842490-3842583-3103138716/socket","Net":"unix"}} +{"time":"2026-05-30T21:29:25.190355799+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-30T21:29:25.199992166+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T21:29:25.505057367+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T21:29:25.973049598+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"9rmldg8z1fv3"} +{"time":"2026-05-30T21:29:26.156360245+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"9rmldg8z1fv3"} +{"time":"2026-05-30T21:29:26.158126937+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T21:29:26.158998856+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-30T21:29:26.159012195+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-30T21:29:26.159026063+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-30T21:29:26.159037503+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-30T21:29:26.159046981+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-30T21:29:26.159075971+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-30T21:29:26.159167957+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-30T21:29:26.159204868+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3842490-3842583-3103138716/socket","Net":"unix"}} +{"time":"2026-05-30T21:29:26.159256504+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260530_212924-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260530_212924-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..dbc8ba45aa876e6ff27af6eda4d114efd1884068 --- /dev/null +++ b/reward/wandb/run-20260530_212924-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-30T21:29:25.20013664+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-30T21:29:25.502286598+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-30T21:29:25.502442236+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-30T21:29:25.502518299+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T21:29:25.504257404+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-30T21:29:25.504310046+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T21:29:25.504376897+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T21:29:25.506504284+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-30T21:29:25.761209483+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-30T21:29:25.972393002+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":24,"current":39} +{"time":"2026-05-30T21:29:26.103615864+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8c64e4b96f4a866c739e1e7194e9a2fad6bdbb01c8f0393e082d9d2a85b43cb3"} +{"time":"2026-05-30T21:29:26.103756035+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B45AF62D14C9184b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_212924-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8c64e4b96f4a866c739e1e7194e9a2fad6bdbb01c8f0393e082d9d2a85b43cb3, Size: 344}"} +{"time":"2026-05-30T21:29:26.105265993+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=097a2e0b6eb92791aa1784d6f83ae8e31cc8e75eee4a57fbf813c97e2fb6e5a8"} +{"time":"2026-05-30T21:29:26.105376372+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B45AF62D1F43A34b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_212924-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=097a2e0b6eb92791aa1784d6f83ae8e31cc8e75eee4a57fbf813c97e2fb6e5a8, Size: 7996}"} +{"time":"2026-05-30T21:29:26.106410548+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4d1980062574209bf826a91e755505237c636a379615d58782dd35abaf103415"} +{"time":"2026-05-30T21:29:26.106471834+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B45AF62E4F80A429678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_212924-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4d1980062574209bf826a91e755505237c636a379615d58782dd35abaf103415, Size: 111}"} +{"time":"2026-05-30T21:29:26.107677187+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c785a46a8ac3d66085d99131ebeb895b4c6c5f1048d8f3f5cd1049d7061c5d46"} +{"time":"2026-05-30T21:29:26.107831597+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B45AF62D4988A74b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_212924-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c785a46a8ac3d66085d99131ebeb895b4c6c5f1048d8f3f5cd1049d7061c5d46, Size: 2982}"} +{"time":"2026-05-30T21:29:26.110085777+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8a7191d28ac83378b9693aab4a780dedaf37d08f72a6000700fa3d165af29164"} +{"time":"2026-05-30T21:29:26.110152147+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B45AF62E7A0E6129678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_212924-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8a7191d28ac83378b9693aab4a780dedaf37d08f72a6000700fa3d165af29164, Size: 2850}"} +{"time":"2026-05-30T21:29:26.112662192+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=79ba85edd08e4c5f494937e57e4dbd7515163f9e21b9a4efbf316974af4ccc58"} +{"time":"2026-05-30T21:29:26.112707059+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B45AF62C94410617e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_212924-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T132926Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=79ba85edd08e4c5f494937e57e4dbd7515163f9e21b9a4efbf316974af4ccc58, Size: 14790}"} +{"time":"2026-05-30T21:29:26.113007763+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-30T21:29:26.154059989+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-30T21:29:26.158164866+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-30T21:29:26.158195366+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T21:29:26.158272738+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-30T21:29:26.158294857+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260530_212924-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260530_212924-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..13a571b73d24521671dc6e8d867ca40204c77a9b --- /dev/null +++ b/reward/wandb/run-20260530_212924-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-30 21:29:24,945 INFO MainThread:3842490 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-30 21:29:24,946 INFO MainThread:3842490 [wandb_setup.py:_flush():81] Configure stats pid to 3842490 +2026-05-30 21:29:24,946 INFO MainThread:3842490 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-30 21:29:24,946 INFO MainThread:3842490 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_212924-fmmkx1a5/logs/debug.log +2026-05-30 21:29:24,946 INFO MainThread:3842490 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_212924-fmmkx1a5/logs/debug-internal.log +2026-05-30 21:29:24,946 INFO MainThread:3842490 [wandb_init.py:init():844] calling init triggers +2026-05-30 21:29:24,946 INFO MainThread:3842490 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 24, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-30 21:29:24,946 INFO MainThread:3842490 [wandb_init.py:init():892] starting backend +2026-05-30 21:29:25,190 INFO MainThread:3842490 [wandb_init.py:init():895] sending inform_init request +2026-05-30 21:29:25,195 INFO MainThread:3842490 [wandb_init.py:init():903] backend started and connected +2026-05-30 21:29:25,197 INFO MainThread:3842490 [wandb_init.py:init():973] updated telemetry +2026-05-30 21:29:25,200 INFO MainThread:3842490 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-30 21:29:25,756 INFO MainThread:3842490 [wandb_init.py:init():1037] run resumed +2026-05-30 21:29:25,759 INFO MainThread:3842490 [wandb_init.py:init():1042] starting run threads in backend +2026-05-30 21:29:25,848 INFO MainThread:3842490 [wandb_run.py:_console_start():2524] atexit reg +2026-05-30 21:29:25,848 INFO MainThread:3842490 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-30 21:29:25,848 INFO MainThread:3842490 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-30 21:29:25,848 INFO MainThread:3842490 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-30 21:29:25,851 INFO MainThread:3842490 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-30 21:29:25,972 INFO MainThread:3842490 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-30 21:29:25,972 INFO MainThread:3842490 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-30 21:29:25,972 INFO MainThread:3842490 [wandb_run.py:_restore():2472] restore +2026-05-30 21:29:25,973 INFO MainThread:3842490 [wandb_run.py:_restore():2478] restore done +2026-05-30 21:29:26,157 INFO MainThread:3842490 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260530_212924-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260530_212924-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..27a7851d0e4ed385f9eaf6d50bcbab8a97266851 Binary files /dev/null and b/reward/wandb/run-20260530_212924-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260530_212930-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260530_212930-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260530_212930-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260530_212930-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260530_212930-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7dbbfa5ac731617ded5e3c26a3364f6772e3fc9 --- /dev/null +++ b/reward/wandb/run-20260530_212930-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + 9dgv1rp4x4lnjncal8zb6qrbtgyu9kn0: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=24 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625597689856" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-30T13:29:30.741626Z" + writerId: 9dgv1rp4x4lnjncal8zb6qrbtgyu9kn0 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 24 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260530_212930-fmmkx1a5/files/output.log b/reward/wandb/run-20260530_212930-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e756902c1b3bd86f2835a53ac5309a3d662efac9 --- /dev/null +++ b/reward/wandb/run-20260530_212930-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 24 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.4551 avg length: 124.63 diff --git a/reward/wandb/run-20260530_212930-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260530_212930-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260530_212930-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260530_212930-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260530_212930-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..efcfa3fb9baee5fece8f44675d38127bdf8cc51f --- /dev/null +++ b/reward/wandb/run-20260530_212930-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/avg_response_length":124.568359375,"train/learning_rate":8.997213817017502e-07,"reward/t_grpo_bonus":0.2718750000000003,"reward/train_step":1,"_step":38,"_wandb":{"runtime":20949},"_runtime":20949,"reward/unbiased_bonus":0.09374999999999996,"reward/acc":0.396484375,"train/loss":-0.3729729354381562,"_timestamp":1.7801299761845806e+09} \ No newline at end of file diff --git a/reward/wandb/run-20260531_024637-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260531_024637-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd2a057b0e648bfbed623c722b58c11ca063e27a --- /dev/null +++ b/reward/wandb/run-20260531_024637-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + 4yqlevbdcw79f3w9ejwvegl8vwgmfr95: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=29 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625681076224" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-30T18:46:37.067852Z" + writerId: 4yqlevbdcw79f3w9ejwvegl8vwgmfr95 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 29 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260531_024637-fmmkx1a5/files/output.log b/reward/wandb/run-20260531_024637-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..407c97243c2812108ae2613836d1eba5898c0fd1 --- /dev/null +++ b/reward/wandb/run-20260531_024637-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 29 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.5078 avg length: 126.24 diff --git a/reward/wandb/run-20260531_024637-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260531_024637-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260531_024637-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260531_024637-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260531_024637-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9a15809319c1b4a4172b2f7c83cc3644979d6f06 --- /dev/null +++ b/reward/wandb/run-20260531_024637-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-30T18:46:37.067852Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=29" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625681076224" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "4yqlevbdcw79f3w9ejwvegl8vwgmfr95" +} \ No newline at end of file diff --git a/reward/wandb/run-20260531_024637-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260531_024637-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fbbe1eb149950130601f091bae4cc036d4a32010 --- /dev/null +++ b/reward/wandb/run-20260531_024637-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":26101,"reward/t_grpo_bonus":0.2718750000000003,"train/learning_rate":8.997213817017502e-07,"train/loss":-0.3729729354381562,"reward/acc":0.396484375,"_step":38,"_wandb":{"runtime":26101},"reward/avg_response_length":124.568359375,"reward/train_step":1,"reward/unbiased_bonus":0.09374999999999996,"_timestamp":1.7801299761845806e+09} \ No newline at end of file diff --git a/reward/wandb/run-20260531_024637-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260531_024637-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..f7629bd6a1149b9ab6e36addc9509a481d57db4e --- /dev/null +++ b/reward/wandb/run-20260531_024637-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-31T02:46:37.132998941+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpvynro5o2/port-4095068.txt","pid":4095068,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-31T02:46:37.133630439+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":4095068} +{"time":"2026-05-31T02:46:37.133618899+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4095068-4095156-2393756756/socket","Net":"unix"}} +{"time":"2026-05-31T02:46:37.314619742+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-31T02:46:37.324202468+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-31T02:46:37.628071286+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-31T02:46:38.098607732+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"vc6x5iv8nnv0"} +{"time":"2026-05-31T02:46:38.280826621+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"vc6x5iv8nnv0"} +{"time":"2026-05-31T02:46:38.282217859+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-31T02:46:38.282908651+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-31T02:46:38.282919784+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-31T02:46:38.282933066+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-31T02:46:38.282943988+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-31T02:46:38.282950273+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-31T02:46:38.283016797+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-4095068-4095156-2393756756/socket","Net":"unix"}} +{"time":"2026-05-31T02:46:38.283062762+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-31T02:46:38.283076199+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-31T02:46:38.283093844+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260531_024637-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260531_024637-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6218c394b1ae71a480237c403e5c24836fe35202 --- /dev/null +++ b/reward/wandb/run-20260531_024637-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-31T02:46:37.324304238+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-31T02:46:37.627541346+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-31T02:46:37.627723829+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-31T02:46:37.627797643+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-31T02:46:37.628053189+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-31T02:46:37.628093425+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-31T02:46:37.628101006+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-31T02:46:37.634051879+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-31T02:46:37.889926005+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-31T02:46:38.097829999+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":29,"current":39} +{"time":"2026-05-31T02:46:38.229484038+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=deae39c1dba658de3737da969e622c35480e3e10cf47671e565049009a2d30f4"} +{"time":"2026-05-31T02:46:38.229619328+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B46C4570BCEF8D29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_024637-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=deae39c1dba658de3737da969e622c35480e3e10cf47671e565049009a2d30f4, Size: 344}"} +{"time":"2026-05-31T02:46:38.229935274+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4444f633db11bbb2831429fdadd9a69797dd6893ec0334fe0284e527de4555d0"} +{"time":"2026-05-31T02:46:38.230059242+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B46C4571610B8A17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_024637-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4444f633db11bbb2831429fdadd9a69797dd6893ec0334fe0284e527de4555d0, Size: 2850}"} +{"time":"2026-05-31T02:46:38.231313404+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3e769a82cf1ae4210b4d4c79db2a0e57883c538e114dc25a758d41495a662477"} +{"time":"2026-05-31T02:46:38.23142287+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B46C4570D6D94F29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_024637-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=3e769a82cf1ae4210b4d4c79db2a0e57883c538e114dc25a758d41495a662477, Size: 2982}"} +{"time":"2026-05-31T02:46:38.231614369+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=73dcb5fd5d29f3bca255dca41db0863001b2024ccf77aeb302eba8936cd03fe1"} +{"time":"2026-05-31T02:46:38.231711722+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B46C45715F28E517e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_024637-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=73dcb5fd5d29f3bca255dca41db0863001b2024ccf77aeb302eba8936cd03fe1, Size: 111}"} +{"time":"2026-05-31T02:46:38.231780327+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=fd1bd0431314f68d35b11a5e6f5a4a83fe203ce34f1363220128a43a7f38c7e8"} +{"time":"2026-05-31T02:46:38.231881348+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B46C45715E8D3917e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_024637-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=fd1bd0431314f68d35b11a5e6f5a4a83fe203ce34f1363220128a43a7f38c7e8, Size: 7996}"} +{"time":"2026-05-31T02:46:38.236832453+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e30cf3babd65cae21a645d4a6979fb93293c669af412aa11920531eaf961f101"} +{"time":"2026-05-31T02:46:38.236873647+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B46C45715E264417e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_024637-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T184638Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e30cf3babd65cae21a645d4a6979fb93293c669af412aa11920531eaf961f101, Size: 14790}"} +{"time":"2026-05-31T02:46:38.237198174+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-31T02:46:38.277275318+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-31T02:46:38.282242139+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-31T02:46:38.282259346+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-31T02:46:38.282324606+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-31T02:46:38.282340064+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260531_024637-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260531_024637-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..96c89929cdb6566aeeb7a0543538c9b357e2586c --- /dev/null +++ b/reward/wandb/run-20260531_024637-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-31 02:46:37,070 INFO MainThread:4095068 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-31 02:46:37,070 INFO MainThread:4095068 [wandb_setup.py:_flush():81] Configure stats pid to 4095068 +2026-05-31 02:46:37,070 INFO MainThread:4095068 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-31 02:46:37,070 INFO MainThread:4095068 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260531_024637-fmmkx1a5/logs/debug.log +2026-05-31 02:46:37,070 INFO MainThread:4095068 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260531_024637-fmmkx1a5/logs/debug-internal.log +2026-05-31 02:46:37,070 INFO MainThread:4095068 [wandb_init.py:init():844] calling init triggers +2026-05-31 02:46:37,070 INFO MainThread:4095068 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 29, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-31 02:46:37,070 INFO MainThread:4095068 [wandb_init.py:init():892] starting backend +2026-05-31 02:46:37,315 INFO MainThread:4095068 [wandb_init.py:init():895] sending inform_init request +2026-05-31 02:46:37,320 INFO MainThread:4095068 [wandb_init.py:init():903] backend started and connected +2026-05-31 02:46:37,321 INFO MainThread:4095068 [wandb_init.py:init():973] updated telemetry +2026-05-31 02:46:37,324 INFO MainThread:4095068 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-31 02:46:37,884 INFO MainThread:4095068 [wandb_init.py:init():1037] run resumed +2026-05-31 02:46:37,887 INFO MainThread:4095068 [wandb_init.py:init():1042] starting run threads in backend +2026-05-31 02:46:37,977 INFO MainThread:4095068 [wandb_run.py:_console_start():2524] atexit reg +2026-05-31 02:46:37,977 INFO MainThread:4095068 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-31 02:46:37,977 INFO MainThread:4095068 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-31 02:46:37,977 INFO MainThread:4095068 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-31 02:46:37,980 INFO MainThread:4095068 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-31 02:46:38,097 INFO MainThread:4095068 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-31 02:46:38,098 INFO MainThread:4095068 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-31 02:46:38,098 INFO MainThread:4095068 [wandb_run.py:_restore():2472] restore +2026-05-31 02:46:38,098 INFO MainThread:4095068 [wandb_run.py:_restore():2478] restore done +2026-05-31 02:46:38,281 INFO MainThread:4095068 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260531_024637-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260531_024637-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..cece21790679a22006dcd29efefb519de9ee2c9c Binary files /dev/null and b/reward/wandb/run-20260531_024637-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260531_024642-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260531_024642-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260531_024642-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260531_024642-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260531_024642-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..155978996901379e2779d30088d21cecd536ebcf --- /dev/null +++ b/reward/wandb/run-20260531_024642-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + 1dfbwhfafyn03tbi4sjmsa5vqry4pazl: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=29 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625681084416" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-30T18:46:42.854913Z" + writerId: 1dfbwhfafyn03tbi4sjmsa5vqry4pazl + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 29 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260531_024642-fmmkx1a5/files/output.log b/reward/wandb/run-20260531_024642-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..407c97243c2812108ae2613836d1eba5898c0fd1 --- /dev/null +++ b/reward/wandb/run-20260531_024642-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 29 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.5078 avg length: 126.24 diff --git a/reward/wandb/run-20260531_024642-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260531_024642-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260531_024642-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260531_034808-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260531_034808-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260531_034808-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260531_034808-fmmkx1a5/files/config.yaml b/reward/wandb/run-20260531_034808-fmmkx1a5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce6b880afb8f571928d0ecc69318bd42d40af24d --- /dev/null +++ b/reward/wandb/run-20260531_034808-fmmkx1a5/files/config.yaml @@ -0,0 +1,281 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + v7hruwldf4qytspjwowi23ou2zsysjym: + args: + - config=../configs/rl_lavida_video_reversal.yaml + - wandb.run_id=fmmkx1a5 + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=30 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "625682087936" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-05-30T19:48:08.194832Z" + writerId: v7hruwldf4qytspjwowi23ou2zsysjym + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_video_reversal.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_rl_train +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 30 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_video_reversal/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_video_reversal +experiment.save_every: + value: 10 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 30 +lr_scheduler.params.learning_rate: + value: 1e-06 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 1e-06 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.01 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: fmmkx1a5 diff --git a/reward/wandb/run-20260531_034808-fmmkx1a5/files/output.log b/reward/wandb/run-20260531_034808-fmmkx1a5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2fcd26171567bf23c26d6dc0a55d2ddc84a870a3 --- /dev/null +++ b/reward/wandb/run-20260531_034808-fmmkx1a5/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 30 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.4473 avg length: 125.82 diff --git a/reward/wandb/run-20260531_034808-fmmkx1a5/files/requirements.txt b/reward/wandb/run-20260531_034808-fmmkx1a5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260531_034808-fmmkx1a5/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260531_034808-fmmkx1a5/files/wandb-metadata.json b/reward/wandb/run-20260531_034808-fmmkx1a5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1b3f48f531df21912e67048633d92d5d0e79a6d4 --- /dev/null +++ b/reward/wandb/run-20260531_034808-fmmkx1a5/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-05-30T19:48:08.194832Z", + "args": [ + "config=../configs/rl_lavida_video_reversal.yaml", + "wandb.run_id=fmmkx1a5", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=30" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "625682087936" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "v7hruwldf4qytspjwowi23ou2zsysjym" +} \ No newline at end of file diff --git a/reward/wandb/run-20260531_034808-fmmkx1a5/files/wandb-summary.json b/reward/wandb/run-20260531_034808-fmmkx1a5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2706cba34ba14a33b54678f22c3e85be682f5736 --- /dev/null +++ b/reward/wandb/run-20260531_034808-fmmkx1a5/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/loss":-0.3729729354381562,"train/learning_rate":8.997213817017502e-07,"reward/unbiased_bonus":0.09374999999999996,"reward/acc":0.396484375,"reward/avg_response_length":124.568359375,"_wandb":{"runtime":27035},"reward/t_grpo_bonus":0.2718750000000003,"reward/train_step":1,"_step":38,"_runtime":27035,"_timestamp":1.7801299761845806e+09} \ No newline at end of file diff --git a/reward/wandb/run-20260531_034808-fmmkx1a5/logs/debug-core.log b/reward/wandb/run-20260531_034808-fmmkx1a5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..f00c69da8c3b6a10c1812d86b2baa830050a4fec --- /dev/null +++ b/reward/wandb/run-20260531_034808-fmmkx1a5/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-05-31T03:48:08.258698954+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpkm826bvi/port-4145072.txt","pid":4145072,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-31T03:48:08.259431174+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":4145072} +{"time":"2026-05-31T03:48:08.259354876+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4145072-4145167-912255097/socket","Net":"unix"}} +{"time":"2026-05-31T03:48:08.440316209+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-31T03:48:08.449273484+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-31T03:48:08.751644161+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-31T03:48:09.165504305+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rgxbfkdpq3yt"} +{"time":"2026-05-31T03:48:09.341314355+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rgxbfkdpq3yt"} +{"time":"2026-05-31T03:48:09.342090373+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-31T03:48:09.342655486+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"} +{"time":"2026-05-31T03:48:09.342662738+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-31T03:48:09.342672705+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-31T03:48:09.342680737+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-31T03:48:09.342695607+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-31T03:48:09.342712498+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-31T03:48:09.342789658+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-31T03:48:09.342876295+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-4145072-4145167-912255097/socket","Net":"unix"}} +{"time":"2026-05-31T03:48:09.342932001+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260531_034808-fmmkx1a5/logs/debug-internal.log b/reward/wandb/run-20260531_034808-fmmkx1a5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..fc01d32f31a28d18a34d0f470d65afb41f8ce0c7 --- /dev/null +++ b/reward/wandb/run-20260531_034808-fmmkx1a5/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-05-31T03:48:08.449458285+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-31T03:48:08.751013467+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-05-31T03:48:08.751155078+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"} +{"time":"2026-05-31T03:48:08.751422965+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-31T03:48:08.751628654+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"} +{"time":"2026-05-31T03:48:08.751647024+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-31T03:48:08.751663424+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"} +{"time":"2026-05-31T03:48:08.755751161+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-05-31T03:48:08.957593953+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-05-31T03:48:09.164805677+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":30,"current":39} +{"time":"2026-05-31T03:48:09.292578116+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=108e4c77cdfb92c227c9ab11a4613970263092fd5cf2c84de7a3f7469577be56"} +{"time":"2026-05-31T03:48:09.293345157+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7e7e2861f2f91eb4ee2d31e2aa924fecf1d69eb488aafa82b6ef7e7c96417fa7"} +{"time":"2026-05-31T03:48:09.294366436+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log18B46FA0D5BD180D17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_034808-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=108e4c77cdfb92c227c9ab11a4613970263092fd5cf2c84de7a3f7469577be56, Size: 111}"} +{"time":"2026-05-31T03:48:09.2938455+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cf8ebe29aacc5012e5c7623297d2aa8526eec0844ab989c53023e95423444cdd"} +{"time":"2026-05-31T03:48:09.294390976+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml18B46FA0D5BDA57E17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_034808-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7e7e2861f2f91eb4ee2d31e2aa924fecf1d69eb488aafa82b6ef7e7c96417fa7, Size: 7996}"} +{"time":"2026-05-31T03:48:09.294524677+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2dfcfb85058959a3e8e2ce09ce7d551078155352322a793311e62c02935984d9"} +{"time":"2026-05-31T03:48:09.294516584+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt18B46FA0D6050E1D4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_034808-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cf8ebe29aacc5012e5c7623297d2aa8526eec0844ab989c53023e95423444cdd, Size: 2982}"} +{"time":"2026-05-31T03:48:09.294689145+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json18B46FA0D6068B304b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_034808-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2dfcfb85058959a3e8e2ce09ce7d551078155352322a793311e62c02935984d9, Size: 344}"} +{"time":"2026-05-31T03:48:09.297719038+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6bb3c77c1d8c69c4a3de69d3f965a1faebd012787e01d87a14cc47e423903115"} +{"time":"2026-05-31T03:48:09.297808493+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json18B46FA0D579FB5429678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_034808-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6bb3c77c1d8c69c4a3de69d3f965a1faebd012787e01d87a14cc47e423903115, Size: 2850}"} +{"time":"2026-05-31T03:48:09.298052675+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6de3165fa3a8260c958e5bdf5cdb6677315f97d7f4ee0fb69ba70b5cb05affab"} +{"time":"2026-05-31T03:48:09.298202806+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py18B46FA0D5A4401D17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260531_034808-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260530%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260530T194809Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6de3165fa3a8260c958e5bdf5cdb6677315f97d7f4ee0fb69ba70b5cb05affab, Size: 14790}"} +{"time":"2026-05-31T03:48:09.298957002+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-31T03:48:09.337418498+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-31T03:48:09.342103758+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"} +{"time":"2026-05-31T03:48:09.342115448+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-31T03:48:09.342163188+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"} +{"time":"2026-05-31T03:48:09.342174826+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"} diff --git a/reward/wandb/run-20260531_034808-fmmkx1a5/logs/debug.log b/reward/wandb/run-20260531_034808-fmmkx1a5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..37043806385f6ca096bc3928d4f202efaa9cd62a --- /dev/null +++ b/reward/wandb/run-20260531_034808-fmmkx1a5/logs/debug.log @@ -0,0 +1,25 @@ +2026-05-31 03:48:08,196 INFO MainThread:4145072 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-31 03:48:08,196 INFO MainThread:4145072 [wandb_setup.py:_flush():81] Configure stats pid to 4145072 +2026-05-31 03:48:08,196 INFO MainThread:4145072 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-31 03:48:08,196 INFO MainThread:4145072 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260531_034808-fmmkx1a5/logs/debug.log +2026-05-31 03:48:08,197 INFO MainThread:4145072 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260531_034808-fmmkx1a5/logs/debug-internal.log +2026-05-31 03:48:08,197 INFO MainThread:4145072 [wandb_init.py:init():844] calling init triggers +2026-05-31 03:48:08,197 INFO MainThread:4145072 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 30, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-05-31 03:48:08,197 INFO MainThread:4145072 [wandb_init.py:init():892] starting backend +2026-05-31 03:48:08,440 INFO MainThread:4145072 [wandb_init.py:init():895] sending inform_init request +2026-05-31 03:48:08,446 INFO MainThread:4145072 [wandb_init.py:init():903] backend started and connected +2026-05-31 03:48:08,447 INFO MainThread:4145072 [wandb_init.py:init():973] updated telemetry +2026-05-31 03:48:08,450 INFO MainThread:4145072 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-31 03:48:08,951 INFO MainThread:4145072 [wandb_init.py:init():1037] run resumed +2026-05-31 03:48:08,955 INFO MainThread:4145072 [wandb_init.py:init():1042] starting run threads in backend +2026-05-31 03:48:09,044 INFO MainThread:4145072 [wandb_run.py:_console_start():2524] atexit reg +2026-05-31 03:48:09,044 INFO MainThread:4145072 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-31 03:48:09,045 INFO MainThread:4145072 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-31 03:48:09,045 INFO MainThread:4145072 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-31 03:48:09,048 INFO MainThread:4145072 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-31 03:48:09,164 INFO MainThread:4145072 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5 +2026-05-31 03:48:09,165 INFO MainThread:4145072 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-05-31 03:48:09,165 INFO MainThread:4145072 [wandb_run.py:_restore():2472] restore +2026-05-31 03:48:09,165 INFO MainThread:4145072 [wandb_run.py:_restore():2478] restore done +2026-05-31 03:48:09,341 INFO MainThread:4145072 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260531_034808-fmmkx1a5/run-fmmkx1a5.wandb b/reward/wandb/run-20260531_034808-fmmkx1a5/run-fmmkx1a5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..133f41b995e426166e559a11ae4900109416c6ce Binary files /dev/null and b/reward/wandb/run-20260531_034808-fmmkx1a5/run-fmmkx1a5.wandb differ diff --git a/reward/wandb/run-20260602_125331-tnrnzimf/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260602_125331-tnrnzimf/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260602_125331-tnrnzimf/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260602_125331-tnrnzimf/files/config.yaml b/reward/wandb/run-20260602_125331-tnrnzimf/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd4b79dff688e7ade1c09f52ff8f908396257dbb --- /dev/null +++ b/reward/wandb/run-20260602_125331-tnrnzimf/files/config.yaml @@ -0,0 +1,221 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + 7qfo6aa726775ttcf04asmzbb23vhwq5: + args: + - config=../configs/rl_lavida_revvideo_orig10k_4gpu.yaml + - wandb.run_id=tnrnzimf + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=1 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + host: b1301b38c75f + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-06-02T04:53:31.559267Z" + writerId: 7qfo6aa726775ttcf04asmzbb23vhwq5 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 13 + - 14 + - 16 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_revvideo_orig10k_4gpu.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_sft_train_cleaned_10k +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 32 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 32 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.steps: + value: 32 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 1 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_revvideo_orig10k_4gpu +experiment.save_every: + value: 5 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 10 +lr_scheduler.params.learning_rate: + value: 5e-07 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/hf/lavida-llada-v1.0-instruct +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 5e-07 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: true +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 32 +rollout.max_gen_length: + value: 32 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 32 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.05 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 32 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: tnrnzimf diff --git a/reward/wandb/run-20260602_125331-tnrnzimf/files/output.log b/reward/wandb/run-20260602_125331-tnrnzimf/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..815b7cba3b96c5f746b9aa3491f516679c04860e --- /dev/null +++ b/reward/wandb/run-20260602_125331-tnrnzimf/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 1 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.0000 avg length: 31.04 diff --git a/reward/wandb/run-20260602_125331-tnrnzimf/files/requirements.txt b/reward/wandb/run-20260602_125331-tnrnzimf/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260602_125331-tnrnzimf/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260602_125331-tnrnzimf/files/wandb-metadata.json b/reward/wandb/run-20260602_125331-tnrnzimf/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2f9f4e8b2def738d0dbd7e754baba7447a65a3c2 --- /dev/null +++ b/reward/wandb/run-20260602_125331-tnrnzimf/files/wandb-metadata.json @@ -0,0 +1,26 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-02T04:53:31.559267Z", + "args": [ + "config=../configs/rl_lavida_revvideo_orig10k_4gpu.yaml", + "wandb.run_id=tnrnzimf", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=1" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "writerId": "7qfo6aa726775ttcf04asmzbb23vhwq5" +} \ No newline at end of file diff --git a/reward/wandb/run-20260602_125331-tnrnzimf/files/wandb-summary.json b/reward/wandb/run-20260602_125331-tnrnzimf/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b47ba8942984ba05f54de889f2e6bc26e82b09be --- /dev/null +++ b/reward/wandb/run-20260602_125331-tnrnzimf/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":0},"_timestamp":1.780376012474483e+09,"reward/unbiased_bonus":0,"_runtime":0.197048589,"reward/train_step":1,"_step":1,"reward/acc":0,"reward/t_grpo_bonus":0.3000000000000004,"reward/avg_response_length":31.0400390625} \ No newline at end of file diff --git a/reward/wandb/run-20260602_125331-tnrnzimf/logs/debug-core.log b/reward/wandb/run-20260602_125331-tnrnzimf/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..cd0321896b84c1a4251f380f9a3129165a283ca9 --- /dev/null +++ b/reward/wandb/run-20260602_125331-tnrnzimf/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-06-02T12:53:31.62436339+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpof2f6sf2/port-672437.txt","pid":672437,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-02T12:53:31.625347755+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":672437} +{"time":"2026-06-02T12:53:31.625335632+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-672437-672620-2059697951/socket","Net":"unix"}} +{"time":"2026-06-02T12:53:31.805939679+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-02T12:53:31.815117865+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"tnrnzimf","id":"1(@)"} +{"time":"2026-06-02T12:53:32.121611547+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"tnrnzimf","id":"1(@)"} +{"time":"2026-06-02T12:53:32.475974164+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"zh7q5061fisv"} +{"time":"2026-06-02T12:53:32.684070415+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"zh7q5061fisv"} +{"time":"2026-06-02T12:53:32.685329095+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"tnrnzimf","id":"1(@)"} +{"time":"2026-06-02T12:53:32.685992837+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"tnrnzimf","id":"1(@)"} +{"time":"2026-06-02T12:53:32.686091434+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-02T12:53:32.686102397+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-02T12:53:32.686108849+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-02T12:53:32.68610995+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-02T12:53:32.686171979+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-672437-672620-2059697951/socket","Net":"unix"}} +{"time":"2026-06-02T12:53:32.686219661+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-02T12:53:32.686235165+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-02T12:53:32.686252309+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260602_125331-tnrnzimf/logs/debug-internal.log b/reward/wandb/run-20260602_125331-tnrnzimf/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..fcf9fa63ca846d1d8674b1c830fa3ee5b79c8410 --- /dev/null +++ b/reward/wandb/run-20260602_125331-tnrnzimf/logs/debug-internal.log @@ -0,0 +1,27 @@ +{"time":"2026-06-02T12:53:31.816516897+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-02T12:53:32.119804396+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-02T12:53:32.119969302+08:00","level":"INFO","msg":"stream: created new stream","id":"tnrnzimf"} +{"time":"2026-06-02T12:53:32.120058971+08:00","level":"INFO","msg":"handler: started","stream_id":"tnrnzimf"} +{"time":"2026-06-02T12:53:32.120238839+08:00","level":"INFO","msg":"stream: started","id":"tnrnzimf"} +{"time":"2026-06-02T12:53:32.120248547+08:00","level":"INFO","msg":"writer: started","stream_id":"tnrnzimf"} +{"time":"2026-06-02T12:53:32.120255801+08:00","level":"INFO","msg":"sender: started","stream_id":"tnrnzimf"} +{"time":"2026-06-02T12:53:32.123223474+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-02T12:53:32.279914189+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-06-02T12:53:32.614363897+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=327ee9841bc72080c269fc646467fe29c0fc143689d60df0e6cda0166cd2c67b"} +{"time":"2026-06-02T12:53:32.614469416+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/requirements.txt18B52A8CF4B4FF9829678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_125331-tnrnzimf/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=327ee9841bc72080c269fc646467fe29c0fc143689d60df0e6cda0166cd2c67b, Size: 2982}"} +{"time":"2026-06-02T12:53:32.614735316+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=282a36c01112c58b88b73ddbf666e73e67ee939df8296e802d0e225793609e18"} +{"time":"2026-06-02T12:53:32.614799913+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/config.yaml18B52A8CF56F860917e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_125331-tnrnzimf/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=282a36c01112c58b88b73ddbf666e73e67ee939df8296e802d0e225793609e18, Size: 5443}"} +{"time":"2026-06-02T12:53:32.617654487+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=366dcfc2dd4d56810c0baa6932d8ebecc5071a1325fb6dffd98a3e54b6d8b13b"} +{"time":"2026-06-02T12:53:32.617812434+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/wandb-summary.json18B52A8CF4C9AAEC29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_125331-tnrnzimf/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=366dcfc2dd4d56810c0baa6932d8ebecc5071a1325fb6dffd98a3e54b6d8b13b, Size: 239}"} +{"time":"2026-06-02T12:53:32.618806523+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5a6750ef11799bb169680a857da55e6f759915d0294b2ecc79970a52f8976211"} +{"time":"2026-06-02T12:53:32.618882318+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/wandb-metadata.json18B52A8CF60629C24b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_125331-tnrnzimf/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5a6750ef11799bb169680a857da55e6f759915d0294b2ecc79970a52f8976211, Size: 982}"} +{"time":"2026-06-02T12:53:32.619006251+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=181196656ce070b8b71c4f8fe36138a7917f210c680a186db3cc51e30a4793a4"} +{"time":"2026-06-02T12:53:32.619089167+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/output.log18B52A8CF60AC3484b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_125331-tnrnzimf/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=181196656ce070b8b71c4f8fe36138a7917f210c680a186db3cc51e30a4793a4, Size: 109}"} +{"time":"2026-06-02T12:53:32.629905524+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=860aa13822034d29829a10ed5b61fd98087386f0d20cbbea4a0a772357cf4441"} +{"time":"2026-06-02T12:53:32.630003951+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/code/reward/rl_reward_video_reversal.py18B52A8CF5065D9729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_125331-tnrnzimf/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T045332Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=860aa13822034d29829a10ed5b61fd98087386f0d20cbbea4a0a772357cf4441, Size: 14790}"} +{"time":"2026-06-02T12:53:32.63082394+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-02T12:53:32.681006884+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-02T12:53:32.685372049+08:00","level":"INFO","msg":"stream: closing","id":"tnrnzimf"} +{"time":"2026-06-02T12:53:32.68539967+08:00","level":"INFO","msg":"handler: closed","stream_id":"tnrnzimf"} +{"time":"2026-06-02T12:53:32.68548187+08:00","level":"INFO","msg":"sender: closed","stream_id":"tnrnzimf"} +{"time":"2026-06-02T12:53:32.685509765+08:00","level":"INFO","msg":"stream: closed","id":"tnrnzimf"} diff --git a/reward/wandb/run-20260602_125331-tnrnzimf/logs/debug.log b/reward/wandb/run-20260602_125331-tnrnzimf/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f35a3bf610e657fe63b3b9a80dcf78419f37976b --- /dev/null +++ b/reward/wandb/run-20260602_125331-tnrnzimf/logs/debug.log @@ -0,0 +1,24 @@ +2026-06-02 12:53:31,561 INFO MainThread:672437 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-02 12:53:31,561 INFO MainThread:672437 [wandb_setup.py:_flush():81] Configure stats pid to 672437 +2026-06-02 12:53:31,561 INFO MainThread:672437 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-02 12:53:31,561 INFO MainThread:672437 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_125331-tnrnzimf/logs/debug.log +2026-06-02 12:53:31,561 INFO MainThread:672437 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_125331-tnrnzimf/logs/debug-internal.log +2026-06-02 12:53:31,561 INFO MainThread:672437 [wandb_init.py:init():844] calling init triggers +2026-06-02 12:53:31,561 INFO MainThread:672437 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'tnrnzimf', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_revvideo_orig10k_4gpu', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 10, 'experiment.save_every': 5, 'experiment.eval_every': 5, 'experiment.current_epoch': 1, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/hf/lavida-llada-v1.0-instruct', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_sft_train_cleaned_10k', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 32, 'rollout.max_gen_length': 32, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 32, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': True, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 32, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.05, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 5e-07, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 5e-07, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 32, 'evaluation.max_gen_length': 32, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 32, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_revvideo_orig10k_4gpu.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-06-02 12:53:31,561 INFO MainThread:672437 [wandb_init.py:init():892] starting backend +2026-06-02 12:53:31,806 INFO MainThread:672437 [wandb_init.py:init():895] sending inform_init request +2026-06-02 12:53:31,811 INFO MainThread:672437 [wandb_init.py:init():903] backend started and connected +2026-06-02 12:53:31,813 INFO MainThread:672437 [wandb_init.py:init():973] updated telemetry +2026-06-02 12:53:31,819 INFO MainThread:672437 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-02 12:53:32,277 INFO MainThread:672437 [wandb_init.py:init():1042] starting run threads in backend +2026-06-02 12:53:32,369 INFO MainThread:672437 [wandb_run.py:_console_start():2524] atexit reg +2026-06-02 12:53:32,369 INFO MainThread:672437 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-02 12:53:32,369 INFO MainThread:672437 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-02 12:53:32,369 INFO MainThread:672437 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-02 12:53:32,372 INFO MainThread:672437 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-02 12:53:32,474 INFO MainThread:672437 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_revvideo_orig10k_4gpu/tnrnzimf +2026-06-02 12:53:32,475 INFO MainThread:672437 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-02 12:53:32,475 INFO MainThread:672437 [wandb_run.py:_restore():2472] restore +2026-06-02 12:53:32,475 INFO MainThread:672437 [wandb_run.py:_restore():2478] restore done +2026-06-02 12:53:32,684 INFO MainThread:672437 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260602_125331-tnrnzimf/run-tnrnzimf.wandb b/reward/wandb/run-20260602_125331-tnrnzimf/run-tnrnzimf.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4159679673c3ee98fe33f673fadc6050f48053bd Binary files /dev/null and b/reward/wandb/run-20260602_125331-tnrnzimf/run-tnrnzimf.wandb differ diff --git a/reward/wandb/run-20260602_132346-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260602_132346-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260602_132346-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260602_132346-w3wdpl8c/files/config.yaml b/reward/wandb/run-20260602_132346-w3wdpl8c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eeb60fff134976cbd67c9057169df8ef097e52de --- /dev/null +++ b/reward/wandb/run-20260602_132346-w3wdpl8c/files/config.yaml @@ -0,0 +1,273 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + opoc8pacrikbldab9zukwzhchj2y97sa: + args: + - config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml + - wandb.run_id=w3wdpl8c + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=1 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "12.8" + disk: + /: + total: "806578724864" + used: "626439544832" + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + gpu: NVIDIA H800 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-2775489f-18c8-2328-8829-c46238575379 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "85520809984" + name: NVIDIA H800 + uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd + host: b1301b38c75f + memory: + total: "2164113764352" + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-06-02T05:23:46.193219Z" + writerId: opoc8pacrikbldab9zukwzhchj2y97sa + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 13 + - 14 + - 16 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_sft_train_cleaned_10k +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 1 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_revvideo_orig10k_4gpu_len128 +experiment.save_every: + value: 5 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 10 +lr_scheduler.params.learning_rate: + value: 5e-07 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/hf/lavida-llada-v1.0-instruct +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 5e-07 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.05 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: w3wdpl8c diff --git a/reward/wandb/run-20260602_132346-w3wdpl8c/files/output.log b/reward/wandb/run-20260602_132346-w3wdpl8c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..06e016c4f89f2b2ff353cdfe26c2a321741aac86 --- /dev/null +++ b/reward/wandb/run-20260602_132346-w3wdpl8c/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 1 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.0273 avg length: 35.73 diff --git a/reward/wandb/run-20260602_132346-w3wdpl8c/files/requirements.txt b/reward/wandb/run-20260602_132346-w3wdpl8c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260602_132346-w3wdpl8c/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260602_132346-w3wdpl8c/files/wandb-metadata.json b/reward/wandb/run-20260602_132346-w3wdpl8c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0a48873b80aa4ac0093d37da4c5dfe9be4ee5b34 --- /dev/null +++ b/reward/wandb/run-20260602_132346-w3wdpl8c/files/wandb-metadata.json @@ -0,0 +1,98 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-02T05:23:46.193219Z", + "args": [ + "config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml", + "wandb.run_id=w3wdpl8c", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=1" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H800", + "gpu_count": 8, + "disk": { + "/": { + "total": "806578724864", + "used": "626439544832" + } + }, + "memory": { + "total": "2164113764352" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f531028f-805c-3301-61b3-0c46329dc9c0" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2775489f-18c8-2328-8829-c46238575379" + }, + { + "name": "NVIDIA H800", + "memoryTotal": "85520809984", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd" + } + ], + "cudaVersion": "12.8", + "writerId": "opoc8pacrikbldab9zukwzhchj2y97sa" +} \ No newline at end of file diff --git a/reward/wandb/run-20260602_132346-w3wdpl8c/files/wandb-summary.json b/reward/wandb/run-20260602_132346-w3wdpl8c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4318a18eace49b74dc1ca0bd7ee01a773b5b4226 --- /dev/null +++ b/reward/wandb/run-20260602_132346-w3wdpl8c/files/wandb-summary.json @@ -0,0 +1 @@ +{"_timestamp":1.780377827197942e+09,"_runtime":0.295091817,"reward/unbiased_bonus":0,"reward/avg_response_length":35.7275390625,"_step":1,"reward/t_grpo_bonus":0.29531250000000037,"_wandb":{"runtime":0},"reward/acc":0.02734375,"reward/train_step":1} \ No newline at end of file diff --git a/reward/wandb/run-20260602_132346-w3wdpl8c/logs/debug-core.log b/reward/wandb/run-20260602_132346-w3wdpl8c/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b4fa33b36549c4c4fcfb9869d5b765da2bf925d9 --- /dev/null +++ b/reward/wandb/run-20260602_132346-w3wdpl8c/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-06-02T13:23:46.260506949+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpnd6iju74/port-748927.txt","pid":748927,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-02T13:23:46.261054341+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":748927} +{"time":"2026-06-02T13:23:46.261032544+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-748927-749116-2025117688/socket","Net":"unix"}} +{"time":"2026-06-02T13:23:46.441877706+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-02T13:23:46.451990015+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T13:23:46.752172791+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T13:23:47.199639478+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6453kv8patij"} +{"time":"2026-06-02T13:23:47.422237186+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6453kv8patij"} +{"time":"2026-06-02T13:23:47.422944587+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T13:23:47.423732807+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T13:23:47.423903496+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-02T13:23:47.423920592+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-02T13:23:47.423931175+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-02T13:23:47.423929549+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-02T13:23:47.423999025+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-02T13:23:47.424011665+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-02T13:23:47.424056178+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-748927-749116-2025117688/socket","Net":"unix"}} +{"time":"2026-06-02T13:23:47.424096536+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260602_132346-w3wdpl8c/logs/debug-internal.log b/reward/wandb/run-20260602_132346-w3wdpl8c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..db4e2c00a3a69d691813350934a934952e4e898b --- /dev/null +++ b/reward/wandb/run-20260602_132346-w3wdpl8c/logs/debug-internal.log @@ -0,0 +1,27 @@ +{"time":"2026-06-02T13:23:46.452212172+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-02T13:23:46.75164581+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-02T13:23:46.751806481+08:00","level":"INFO","msg":"stream: created new stream","id":"w3wdpl8c"} +{"time":"2026-06-02T13:23:46.751898826+08:00","level":"INFO","msg":"handler: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T13:23:46.752152551+08:00","level":"INFO","msg":"stream: started","id":"w3wdpl8c"} +{"time":"2026-06-02T13:23:46.752187506+08:00","level":"INFO","msg":"writer: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T13:23:46.752236787+08:00","level":"INFO","msg":"sender: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T13:23:46.754279871+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-02T13:23:46.905452898+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-06-02T13:23:47.35351688+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=086d785206a0acfd8246974a214a7bcb25d68f8d4cc74760663b52961f513aca"} +{"time":"2026-06-02T13:23:47.353651813+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml18B52C337C9BA8664b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_132346-w3wdpl8c/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=086d785206a0acfd8246974a214a7bcb25d68f8d4cc74760663b52961f513aca, Size: 7790}"} +{"time":"2026-06-02T13:23:47.356321514+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=75fb2a085ac28ab6a572f83de132c53da94514befd7f00bd8a5e5e1c49291b40"} +{"time":"2026-06-02T13:23:47.356307447+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=bf6f4399f74da432fb1aca7de6c80e5eb206d79d61c1b536014f7c0e4b202569"} +{"time":"2026-06-02T13:23:47.356368893+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt18B52C337CB914544b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_132346-w3wdpl8c/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=75fb2a085ac28ab6a572f83de132c53da94514befd7f00bd8a5e5e1c49291b40, Size: 2982}"} +{"time":"2026-06-02T13:23:47.356515218+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json18B52C337C5417E417e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_132346-w3wdpl8c/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=bf6f4399f74da432fb1aca7de6c80e5eb206d79d61c1b536014f7c0e4b202569, Size: 249}"} +{"time":"2026-06-02T13:23:47.356432748+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=01a346e342105710042c67affa384f57c09b4270098ec98d22fe2281693dc4f3"} +{"time":"2026-06-02T13:23:47.356709398+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log18B52C337C57D18B17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_132346-w3wdpl8c/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=01a346e342105710042c67affa384f57c09b4270098ec98d22fe2281693dc4f3, Size: 109}"} +{"time":"2026-06-02T13:23:47.357249833+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d97b1b9be91a0c3871c5e8362c34f589c1d28a79659f31678ed2e079dbfd3b60"} +{"time":"2026-06-02T13:23:47.357458353+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json18B52C337C52F22917e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_132346-w3wdpl8c/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d97b1b9be91a0c3871c5e8362c34f589c1d28a79659f31678ed2e079dbfd3b60, Size: 2863}"} +{"time":"2026-06-02T13:23:47.362670218+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4acaa098df48774d91a19a6c12b3a998c504f43f17f1d0aaa4977aa26202d9ef"} +{"time":"2026-06-02T13:23:47.362815992+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py18B52C337C533A9317e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_132346-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T052347Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=4acaa098df48774d91a19a6c12b3a998c504f43f17f1d0aaa4977aa26202d9ef, Size: 14790}"} +{"time":"2026-06-02T13:23:47.363105059+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-02T13:23:47.417576005+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-02T13:23:47.422995262+08:00","level":"INFO","msg":"stream: closing","id":"w3wdpl8c"} +{"time":"2026-06-02T13:23:47.423065809+08:00","level":"INFO","msg":"handler: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T13:23:47.423164543+08:00","level":"INFO","msg":"sender: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T13:23:47.42318452+08:00","level":"INFO","msg":"stream: closed","id":"w3wdpl8c"} diff --git a/reward/wandb/run-20260602_132346-w3wdpl8c/logs/debug.log b/reward/wandb/run-20260602_132346-w3wdpl8c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b9915177744991f7bfcde5807b0505c4b85ce96a --- /dev/null +++ b/reward/wandb/run-20260602_132346-w3wdpl8c/logs/debug.log @@ -0,0 +1,24 @@ +2026-06-02 13:23:46,196 INFO MainThread:748927 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-02 13:23:46,196 INFO MainThread:748927 [wandb_setup.py:_flush():81] Configure stats pid to 748927 +2026-06-02 13:23:46,196 INFO MainThread:748927 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-02 13:23:46,196 INFO MainThread:748927 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_132346-w3wdpl8c/logs/debug.log +2026-06-02 13:23:46,196 INFO MainThread:748927 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_132346-w3wdpl8c/logs/debug-internal.log +2026-06-02 13:23:46,196 INFO MainThread:748927 [wandb_init.py:init():844] calling init triggers +2026-06-02 13:23:46,196 INFO MainThread:748927 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'w3wdpl8c', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_revvideo_orig10k_4gpu_len128', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 10, 'experiment.save_every': 5, 'experiment.eval_every': 5, 'experiment.current_epoch': 1, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/hf/lavida-llada-v1.0-instruct', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_sft_train_cleaned_10k', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.05, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 5e-07, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 5e-07, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-06-02 13:23:46,197 INFO MainThread:748927 [wandb_init.py:init():892] starting backend +2026-06-02 13:23:46,442 INFO MainThread:748927 [wandb_init.py:init():895] sending inform_init request +2026-06-02 13:23:46,447 INFO MainThread:748927 [wandb_init.py:init():903] backend started and connected +2026-06-02 13:23:46,449 INFO MainThread:748927 [wandb_init.py:init():973] updated telemetry +2026-06-02 13:23:46,452 INFO MainThread:748927 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-02 13:23:46,903 INFO MainThread:748927 [wandb_init.py:init():1042] starting run threads in backend +2026-06-02 13:23:46,997 INFO MainThread:748927 [wandb_run.py:_console_start():2524] atexit reg +2026-06-02 13:23:46,997 INFO MainThread:748927 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-02 13:23:46,997 INFO MainThread:748927 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-02 13:23:46,997 INFO MainThread:748927 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-02 13:23:47,002 INFO MainThread:748927 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-02 13:23:47,198 INFO MainThread:748927 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c +2026-06-02 13:23:47,198 INFO MainThread:748927 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-02 13:23:47,199 INFO MainThread:748927 [wandb_run.py:_restore():2472] restore +2026-06-02 13:23:47,199 INFO MainThread:748927 [wandb_run.py:_restore():2478] restore done +2026-06-02 13:23:47,421 INFO MainThread:748927 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260602_132346-w3wdpl8c/run-w3wdpl8c.wandb b/reward/wandb/run-20260602_132346-w3wdpl8c/run-w3wdpl8c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d6d8d1abfe1036d7ad10ec900611f4ad2583b266 Binary files /dev/null and b/reward/wandb/run-20260602_132346-w3wdpl8c/run-w3wdpl8c.wandb differ diff --git a/reward/wandb/run-20260602_140728-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260602_140728-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260602_140728-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260602_140728-w3wdpl8c/files/config.yaml b/reward/wandb/run-20260602_140728-w3wdpl8c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d94d756b5932b655edabb510129e26f30e533752 --- /dev/null +++ b/reward/wandb/run-20260602_140728-w3wdpl8c/files/config.yaml @@ -0,0 +1,229 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + mat11aa3yvh6r6khnp0t4f3nvnp97rz7: + args: + - config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml + - wandb.run_id=w3wdpl8c + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=3 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + host: b1301b38c75f + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-06-02T06:07:28.454430Z" + writerId: mat11aa3yvh6r6khnp0t4f3nvnp97rz7 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_sft_train_cleaned_10k +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 3 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_revvideo_orig10k_4gpu_len128/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_revvideo_orig10k_4gpu_len128 +experiment.save_every: + value: 5 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 10 +lr_scheduler.params.learning_rate: + value: 5e-07 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/hf/lavida-llada-v1.0-instruct +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 5e-07 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.05 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: w3wdpl8c diff --git a/reward/wandb/run-20260602_140728-w3wdpl8c/files/output.log b/reward/wandb/run-20260602_140728-w3wdpl8c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2dd695639ae08243e578e205b1b46c98b2fa43ab --- /dev/null +++ b/reward/wandb/run-20260602_140728-w3wdpl8c/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 3 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.0352 avg length: 35.44 diff --git a/reward/wandb/run-20260602_140728-w3wdpl8c/files/requirements.txt b/reward/wandb/run-20260602_140728-w3wdpl8c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260602_140728-w3wdpl8c/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260602_140728-w3wdpl8c/files/wandb-metadata.json b/reward/wandb/run-20260602_140728-w3wdpl8c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..23dad68fc670872c2e9591dfc60017a12371218d --- /dev/null +++ b/reward/wandb/run-20260602_140728-w3wdpl8c/files/wandb-metadata.json @@ -0,0 +1,26 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-02T06:07:28.454430Z", + "args": [ + "config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml", + "wandb.run_id=w3wdpl8c", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=3" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "writerId": "mat11aa3yvh6r6khnp0t4f3nvnp97rz7" +} \ No newline at end of file diff --git a/reward/wandb/run-20260602_140728-w3wdpl8c/files/wandb-summary.json b/reward/wandb/run-20260602_140728-w3wdpl8c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..576467cbff7ebf378e99e548e98e6a7087c29a81 --- /dev/null +++ b/reward/wandb/run-20260602_140728-w3wdpl8c/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/acc":0.03515625,"reward/train_step":3,"reward/t_grpo_bonus":0.3000000000000004,"_wandb":{"runtime":725},"reward/avg_response_length":35.435546875,"train/loss":-0.5786913633346558,"train/learning_rate":1.249999999999999e-07,"_runtime":725.21013027,"reward/unbiased_bonus":0.01875,"_timestamp":1.7803804497481031e+09,"_step":3} \ No newline at end of file diff --git a/reward/wandb/run-20260602_140728-w3wdpl8c/logs/debug-core.log b/reward/wandb/run-20260602_140728-w3wdpl8c/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..156791052684bb23f9a2403ad018f5e4022d31df --- /dev/null +++ b/reward/wandb/run-20260602_140728-w3wdpl8c/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-06-02T14:07:28.520597972+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpgn8lvxz2/port-864183.txt","pid":864183,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-02T14:07:28.521517583+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":864183} +{"time":"2026-06-02T14:07:28.521515115+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-864183-864391-1628080521/socket","Net":"unix"}} +{"time":"2026-06-02T14:07:28.701677902+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-02T14:07:28.710214896+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T14:07:29.311403558+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T14:07:29.749576478+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"cixw5l94n3ig"} +{"time":"2026-06-02T14:07:29.936371398+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"cixw5l94n3ig"} +{"time":"2026-06-02T14:07:29.938913287+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T14:07:29.939589234+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T14:07:29.939686283+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-02T14:07:29.939713811+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-02T14:07:29.939724286+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-02T14:07:29.939755046+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-02T14:07:29.939761923+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-02T14:07:29.939769238+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-02T14:07:29.939829237+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-864183-864391-1628080521/socket","Net":"unix"}} +{"time":"2026-06-02T14:07:29.939848505+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260602_140728-w3wdpl8c/logs/debug-internal.log b/reward/wandb/run-20260602_140728-w3wdpl8c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..08885585c05ac542365a1886ca7ad35b2abb37cf --- /dev/null +++ b/reward/wandb/run-20260602_140728-w3wdpl8c/logs/debug-internal.log @@ -0,0 +1,27 @@ +{"time":"2026-06-02T14:07:28.710376472+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-02T14:07:29.31087523+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-02T14:07:29.311056685+08:00","level":"INFO","msg":"stream: created new stream","id":"w3wdpl8c"} +{"time":"2026-06-02T14:07:29.31113619+08:00","level":"INFO","msg":"handler: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T14:07:29.311382868+08:00","level":"INFO","msg":"stream: started","id":"w3wdpl8c"} +{"time":"2026-06-02T14:07:29.311424302+08:00","level":"INFO","msg":"writer: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T14:07:29.311421912+08:00","level":"INFO","msg":"sender: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T14:07:29.315780542+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-02T14:07:29.540230185+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-06-02T14:07:29.871823495+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5b68d5e9d91824b17737fcee44b9e0887f92857c6f0df6aab2d780886d007752"} +{"time":"2026-06-02T14:07:29.871915143+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml18B52E96166AD5E517e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_140728-w3wdpl8c/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5b68d5e9d91824b17737fcee44b9e0887f92857c6f0df6aab2d780886d007752, Size: 5725}"} +{"time":"2026-06-02T14:07:29.874269104+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a863d333c0def1c7bd8d1f7f3e5d73adcbfbc96e0800ecb2438f49ed96d9ce37"} +{"time":"2026-06-02T14:07:29.8743791+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt18B52E9616F197494b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_140728-w3wdpl8c/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a863d333c0def1c7bd8d1f7f3e5d73adcbfbc96e0800ecb2438f49ed96d9ce37, Size: 2982}"} +{"time":"2026-06-02T14:07:29.874427674+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0f89bcb9a3c3fee48c2eff7fe963b4a3e62c2eb1d5a0ab52213707ce4c3da55b"} +{"time":"2026-06-02T14:07:29.874563231+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json18B52E9616829CBD17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_140728-w3wdpl8c/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0f89bcb9a3c3fee48c2eff7fe963b4a3e62c2eb1d5a0ab52213707ce4c3da55b, Size: 334}"} +{"time":"2026-06-02T14:07:29.876935342+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5c37289f8f118d799157f7f071cc101a43d10592854963017854100c8271699a"} +{"time":"2026-06-02T14:07:29.878341396+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0988cbcded5380c19998260939397f83bbe857ddbb291447611ec134b0f4b0ab"} +{"time":"2026-06-02T14:07:29.880234262+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log18B52E9615DF183629678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_140728-w3wdpl8c/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5c37289f8f118d799157f7f071cc101a43d10592854963017854100c8271699a, Size: 109}"} +{"time":"2026-06-02T14:07:29.880268355+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json18B52E9615E6697D29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_140728-w3wdpl8c/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0988cbcded5380c19998260939397f83bbe857ddbb291447611ec134b0f4b0ab, Size: 989}"} +{"time":"2026-06-02T14:07:29.890324449+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9b2faae217262a0cc2b35ccef5ae4d68dea9288a5da9a5d69116b1c2746faa36"} +{"time":"2026-06-02T14:07:29.890451633+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py18B52E9615CB475A29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_140728-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T060729Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9b2faae217262a0cc2b35ccef5ae4d68dea9288a5da9a5d69116b1c2746faa36, Size: 14790}"} +{"time":"2026-06-02T14:07:29.891007097+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-02T14:07:29.933627398+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-02T14:07:29.938953414+08:00","level":"INFO","msg":"stream: closing","id":"w3wdpl8c"} +{"time":"2026-06-02T14:07:29.938996912+08:00","level":"INFO","msg":"handler: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T14:07:29.939064046+08:00","level":"INFO","msg":"sender: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T14:07:29.939076611+08:00","level":"INFO","msg":"stream: closed","id":"w3wdpl8c"} diff --git a/reward/wandb/run-20260602_140728-w3wdpl8c/logs/debug.log b/reward/wandb/run-20260602_140728-w3wdpl8c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b25364985ffa39395d36c1d8f284de25551ff71c --- /dev/null +++ b/reward/wandb/run-20260602_140728-w3wdpl8c/logs/debug.log @@ -0,0 +1,25 @@ +2026-06-02 14:07:28,457 INFO MainThread:864183 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-02 14:07:28,457 INFO MainThread:864183 [wandb_setup.py:_flush():81] Configure stats pid to 864183 +2026-06-02 14:07:28,457 INFO MainThread:864183 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-02 14:07:28,457 INFO MainThread:864183 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_140728-w3wdpl8c/logs/debug.log +2026-06-02 14:07:28,457 INFO MainThread:864183 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_140728-w3wdpl8c/logs/debug-internal.log +2026-06-02 14:07:28,457 INFO MainThread:864183 [wandb_init.py:init():844] calling init triggers +2026-06-02 14:07:28,457 INFO MainThread:864183 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'w3wdpl8c', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_revvideo_orig10k_4gpu_len128', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 10, 'experiment.save_every': 5, 'experiment.eval_every': 5, 'experiment.current_epoch': 3, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/hf/lavida-llada-v1.0-instruct', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_sft_train_cleaned_10k', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.05, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 5e-07, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 5e-07, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-06-02 14:07:28,457 INFO MainThread:864183 [wandb_init.py:init():892] starting backend +2026-06-02 14:07:28,702 INFO MainThread:864183 [wandb_init.py:init():895] sending inform_init request +2026-06-02 14:07:28,707 INFO MainThread:864183 [wandb_init.py:init():903] backend started and connected +2026-06-02 14:07:28,708 INFO MainThread:864183 [wandb_init.py:init():973] updated telemetry +2026-06-02 14:07:28,711 INFO MainThread:864183 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-02 14:07:29,534 INFO MainThread:864183 [wandb_init.py:init():1037] run resumed +2026-06-02 14:07:29,538 INFO MainThread:864183 [wandb_init.py:init():1042] starting run threads in backend +2026-06-02 14:07:29,627 INFO MainThread:864183 [wandb_run.py:_console_start():2524] atexit reg +2026-06-02 14:07:29,627 INFO MainThread:864183 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-02 14:07:29,627 INFO MainThread:864183 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-02 14:07:29,627 INFO MainThread:864183 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-02 14:07:29,631 INFO MainThread:864183 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-02 14:07:29,748 INFO MainThread:864183 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c +2026-06-02 14:07:29,748 INFO MainThread:864183 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-02 14:07:29,749 INFO MainThread:864183 [wandb_run.py:_restore():2472] restore +2026-06-02 14:07:29,749 INFO MainThread:864183 [wandb_run.py:_restore():2478] restore done +2026-06-02 14:07:29,937 INFO MainThread:864183 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260602_140728-w3wdpl8c/run-w3wdpl8c.wandb b/reward/wandb/run-20260602_140728-w3wdpl8c/run-w3wdpl8c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4be0018b2033a7d7eecb148a52837f7f75911e3a Binary files /dev/null and b/reward/wandb/run-20260602_140728-w3wdpl8c/run-w3wdpl8c.wandb differ diff --git a/reward/wandb/run-20260602_152755-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260602_152755-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260602_152755-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260602_152755-w3wdpl8c/files/config.yaml b/reward/wandb/run-20260602_152755-w3wdpl8c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e05eea487e1a8b537910134120a87846f5e4636b --- /dev/null +++ b/reward/wandb/run-20260602_152755-w3wdpl8c/files/config.yaml @@ -0,0 +1,229 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + 87mgpsd36ulkn5vi2parnnpnzqumwrkx: + args: + - config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml + - wandb.run_id=w3wdpl8c + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=6 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + host: b1301b38c75f + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-06-02T07:27:55.155348Z" + writerId: 87mgpsd36ulkn5vi2parnnpnzqumwrkx + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_sft_train_cleaned_10k +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 6 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_revvideo_orig10k_4gpu_len128/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_revvideo_orig10k_4gpu_len128 +experiment.save_every: + value: 5 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 10 +lr_scheduler.params.learning_rate: + value: 5e-07 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/hf/lavida-llada-v1.0-instruct +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 5e-07 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.05 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: w3wdpl8c diff --git a/reward/wandb/run-20260602_152755-w3wdpl8c/files/output.log b/reward/wandb/run-20260602_152755-w3wdpl8c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..03ab96c8d5232f91ca5203423e9ff3178a4c532d --- /dev/null +++ b/reward/wandb/run-20260602_152755-w3wdpl8c/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 6 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.0410 avg length: 39.33 diff --git a/reward/wandb/run-20260602_152755-w3wdpl8c/files/requirements.txt b/reward/wandb/run-20260602_152755-w3wdpl8c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260602_152755-w3wdpl8c/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260602_152755-w3wdpl8c/files/wandb-metadata.json b/reward/wandb/run-20260602_152755-w3wdpl8c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..68edbc51e64643b3a6b3d3e84fb9487dce56797d --- /dev/null +++ b/reward/wandb/run-20260602_152755-w3wdpl8c/files/wandb-metadata.json @@ -0,0 +1,26 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-02T07:27:55.155348Z", + "args": [ + "config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml", + "wandb.run_id=w3wdpl8c", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=6" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "writerId": "87mgpsd36ulkn5vi2parnnpnzqumwrkx" +} \ No newline at end of file diff --git a/reward/wandb/run-20260602_152755-w3wdpl8c/files/wandb-summary.json b/reward/wandb/run-20260602_152755-w3wdpl8c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..259a25d0154d14aac85d93bfcfb12b19accc0617 --- /dev/null +++ b/reward/wandb/run-20260602_152755-w3wdpl8c/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/t_grpo_bonus":0.3000000000000004,"_runtime":2339,"reward/acc":0.03515625,"_step":10,"_wandb":{"runtime":2339},"reward/unbiased_bonus":0.01875,"train/learning_rate":8.628481651367888e-08,"train/loss":0.3788445591926574,"reward/avg_response_length":35.435546875,"reward/train_step":3,"_timestamp":1.780382672555827e+09} \ No newline at end of file diff --git a/reward/wandb/run-20260602_152755-w3wdpl8c/logs/debug-core.log b/reward/wandb/run-20260602_152755-w3wdpl8c/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..95e577572ff0b5cea35a8380d5ca7d1e0cf5f090 --- /dev/null +++ b/reward/wandb/run-20260602_152755-w3wdpl8c/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-06-02T15:27:55.241072459+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpq43sk8qf/port-1060725.txt","pid":1060725,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-02T15:27:55.241979292+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1060725} +{"time":"2026-06-02T15:27:55.241949536+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1060725-1060910-895001094/socket","Net":"unix"}} +{"time":"2026-06-02T15:27:55.413133438+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-02T15:27:55.422637939+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:27:56.444347217+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:27:56.970010077+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"uqig3khe52yl"} +{"time":"2026-06-02T15:27:57.16697127+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"uqig3khe52yl"} +{"time":"2026-06-02T15:27:57.167900847+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:27:57.168759644+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:27:57.169247738+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-02T15:27:57.16932437+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-02T15:27:57.169359721+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-02T15:27:57.169341926+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-02T15:27:57.16947724+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-02T15:27:57.169499917+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-02T15:27:57.169566602+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1060725-1060910-895001094/socket","Net":"unix"}} +{"time":"2026-06-02T15:27:57.169635588+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260602_152755-w3wdpl8c/logs/debug-internal.log b/reward/wandb/run-20260602_152755-w3wdpl8c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e7494fba622ae39cba9c2743950bafce0bb03ef2 --- /dev/null +++ b/reward/wandb/run-20260602_152755-w3wdpl8c/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-06-02T15:27:55.422847947+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-02T15:27:56.443611951+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-02T15:27:56.443864349+08:00","level":"INFO","msg":"stream: created new stream","id":"w3wdpl8c"} +{"time":"2026-06-02T15:27:56.443982026+08:00","level":"INFO","msg":"handler: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:27:56.444307822+08:00","level":"INFO","msg":"stream: started","id":"w3wdpl8c"} +{"time":"2026-06-02T15:27:56.444339027+08:00","level":"INFO","msg":"writer: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:27:56.444353993+08:00","level":"INFO","msg":"sender: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:27:56.446622323+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-02T15:27:56.674037775+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-06-02T15:27:56.968363667+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":6,"current":11} +{"time":"2026-06-02T15:27:57.107600129+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=dba8a9ef696ec00124f81c65b0ca072f57f823ed491cb0d2eaaf8f125a954624"} +{"time":"2026-06-02T15:27:57.107774367+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json18B532FA03FE5C0817e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_152755-w3wdpl8c/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=dba8a9ef696ec00124f81c65b0ca072f57f823ed491cb0d2eaaf8f125a954624, Size: 989}"} +{"time":"2026-06-02T15:27:57.108767768+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a34b0138077eef759e1346ba80d17250f12cd2a2a8a67d81b8ddff08c87f2754"} +{"time":"2026-06-02T15:27:57.1088805+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt18B532FA0349BA2729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_152755-w3wdpl8c/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a34b0138077eef759e1346ba80d17250f12cd2a2a8a67d81b8ddff08c87f2754, Size: 2982}"} +{"time":"2026-06-02T15:27:57.109155972+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=43bb9923d910913a79dc56f3681ae9752a368a781abcbace008f5142ba59845e"} +{"time":"2026-06-02T15:27:57.10942356+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log18B532FA034F55EE29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_152755-w3wdpl8c/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=43bb9923d910913a79dc56f3681ae9752a368a781abcbace008f5142ba59845e, Size: 109}"} +{"time":"2026-06-02T15:27:57.109969172+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a90df51a32c08118e09bb03d9306bca452188bcd2da82b99877f4888773b96ee"} +{"time":"2026-06-02T15:27:57.110126972+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml18B532FA042463EB17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_152755-w3wdpl8c/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a90df51a32c08118e09bb03d9306bca452188bcd2da82b99877f4888773b96ee, Size: 5725}"} +{"time":"2026-06-02T15:27:57.112717788+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0b61b8a1344f598f4ac33b8be9e62bc37e9082f7a79401af67f7b83f655f1119"} +{"time":"2026-06-02T15:27:57.112823813+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json18B532FA042E3B9C17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_152755-w3wdpl8c/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0b61b8a1344f598f4ac33b8be9e62bc37e9082f7a79401af67f7b83f655f1119, Size: 326}"} +{"time":"2026-06-02T15:27:57.117090964+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=03f9647998d15738a8076ac59a6af60134e454cbc7cb5182d245c47bfa82ff07"} +{"time":"2026-06-02T15:27:57.117214623+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py18B532FA047BA8AF4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_152755-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T072757Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=03f9647998d15738a8076ac59a6af60134e454cbc7cb5182d245c47bfa82ff07, Size: 14790}"} +{"time":"2026-06-02T15:27:57.117520077+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-02T15:27:57.16140763+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-02T15:27:57.167952213+08:00","level":"INFO","msg":"stream: closing","id":"w3wdpl8c"} +{"time":"2026-06-02T15:27:57.167985845+08:00","level":"INFO","msg":"handler: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:27:57.168096524+08:00","level":"INFO","msg":"sender: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:27:57.16811007+08:00","level":"INFO","msg":"stream: closed","id":"w3wdpl8c"} diff --git a/reward/wandb/run-20260602_152755-w3wdpl8c/logs/debug.log b/reward/wandb/run-20260602_152755-w3wdpl8c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..db61609256371afd063b12bde753246812bda40a --- /dev/null +++ b/reward/wandb/run-20260602_152755-w3wdpl8c/logs/debug.log @@ -0,0 +1,25 @@ +2026-06-02 15:27:55,157 INFO MainThread:1060725 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-02 15:27:55,157 INFO MainThread:1060725 [wandb_setup.py:_flush():81] Configure stats pid to 1060725 +2026-06-02 15:27:55,157 INFO MainThread:1060725 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-02 15:27:55,157 INFO MainThread:1060725 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_152755-w3wdpl8c/logs/debug.log +2026-06-02 15:27:55,157 INFO MainThread:1060725 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_152755-w3wdpl8c/logs/debug-internal.log +2026-06-02 15:27:55,157 INFO MainThread:1060725 [wandb_init.py:init():844] calling init triggers +2026-06-02 15:27:55,157 INFO MainThread:1060725 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'w3wdpl8c', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_revvideo_orig10k_4gpu_len128', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 10, 'experiment.save_every': 5, 'experiment.eval_every': 5, 'experiment.current_epoch': 6, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/hf/lavida-llada-v1.0-instruct', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_sft_train_cleaned_10k', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.05, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 5e-07, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 5e-07, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-06-02 15:27:55,157 INFO MainThread:1060725 [wandb_init.py:init():892] starting backend +2026-06-02 15:27:55,413 INFO MainThread:1060725 [wandb_init.py:init():895] sending inform_init request +2026-06-02 15:27:55,420 INFO MainThread:1060725 [wandb_init.py:init():903] backend started and connected +2026-06-02 15:27:55,422 INFO MainThread:1060725 [wandb_init.py:init():973] updated telemetry +2026-06-02 15:27:55,428 INFO MainThread:1060725 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-02 15:27:56,669 INFO MainThread:1060725 [wandb_init.py:init():1037] run resumed +2026-06-02 15:27:56,672 INFO MainThread:1060725 [wandb_init.py:init():1042] starting run threads in backend +2026-06-02 15:27:56,782 INFO MainThread:1060725 [wandb_run.py:_console_start():2524] atexit reg +2026-06-02 15:27:56,782 INFO MainThread:1060725 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-02 15:27:56,782 INFO MainThread:1060725 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-02 15:27:56,783 INFO MainThread:1060725 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-02 15:27:56,788 INFO MainThread:1060725 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-02 15:27:56,968 INFO MainThread:1060725 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c +2026-06-02 15:27:56,968 INFO MainThread:1060725 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-02 15:27:56,969 INFO MainThread:1060725 [wandb_run.py:_restore():2472] restore +2026-06-02 15:27:56,970 INFO MainThread:1060725 [wandb_run.py:_restore():2478] restore done +2026-06-02 15:27:57,166 INFO MainThread:1060725 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260602_152755-w3wdpl8c/run-w3wdpl8c.wandb b/reward/wandb/run-20260602_152755-w3wdpl8c/run-w3wdpl8c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5141a3cb017fe94572691a7694aea09a6037953e Binary files /dev/null and b/reward/wandb/run-20260602_152755-w3wdpl8c/run-w3wdpl8c.wandb differ diff --git a/reward/wandb/run-20260602_155753-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260602_155753-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260602_155753-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260602_155753-w3wdpl8c/files/config.yaml b/reward/wandb/run-20260602_155753-w3wdpl8c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..080dea4521b7d2bdc07a1cd7ac266a128cd5e7ee --- /dev/null +++ b/reward/wandb/run-20260602_155753-w3wdpl8c/files/config.yaml @@ -0,0 +1,229 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + o0pg1kmrvuplnms3lxhcz289cwpn4bhg: + args: + - config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml + - wandb.run_id=w3wdpl8c + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=7 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + host: b1301b38c75f + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-06-02T07:57:53.498293Z" + writerId: o0pg1kmrvuplnms3lxhcz289cwpn4bhg + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_sft_train_cleaned_10k +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 7 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_revvideo_orig10k_4gpu_len128/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_revvideo_orig10k_4gpu_len128 +experiment.save_every: + value: 5 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 10 +lr_scheduler.params.learning_rate: + value: 5e-07 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/hf/lavida-llada-v1.0-instruct +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 5e-07 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.05 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: w3wdpl8c diff --git a/reward/wandb/run-20260602_155753-w3wdpl8c/files/output.log b/reward/wandb/run-20260602_155753-w3wdpl8c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6177ff0947e107d2878a505795d61f1ed9fc0911 --- /dev/null +++ b/reward/wandb/run-20260602_155753-w3wdpl8c/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 7 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.0410 avg length: 37.15 diff --git a/reward/wandb/run-20260602_155753-w3wdpl8c/files/requirements.txt b/reward/wandb/run-20260602_155753-w3wdpl8c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260602_155753-w3wdpl8c/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260602_155753-w3wdpl8c/files/wandb-metadata.json b/reward/wandb/run-20260602_155753-w3wdpl8c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..dd430fecebe4e5bf181417ff7f119a34eac728e9 --- /dev/null +++ b/reward/wandb/run-20260602_155753-w3wdpl8c/files/wandb-metadata.json @@ -0,0 +1,26 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-02T07:57:53.498293Z", + "args": [ + "config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml", + "wandb.run_id=w3wdpl8c", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=7" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "writerId": "o0pg1kmrvuplnms3lxhcz289cwpn4bhg" +} \ No newline at end of file diff --git a/reward/wandb/run-20260602_155753-w3wdpl8c/files/wandb-summary.json b/reward/wandb/run-20260602_155753-w3wdpl8c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0939d1e63e98b9a4f523d5bb92b208b950156ba5 --- /dev/null +++ b/reward/wandb/run-20260602_155753-w3wdpl8c/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/unbiased_bonus":0.01875,"reward/t_grpo_bonus":0.3000000000000004,"train/learning_rate":8.628481651367888e-08,"_step":10,"reward/acc":0.03515625,"_wandb":{"runtime":3084},"_runtime":3084,"reward/avg_response_length":35.435546875,"_timestamp":1.780382672555827e+09,"train/loss":0.3788445591926574,"reward/train_step":3} \ No newline at end of file diff --git a/reward/wandb/run-20260602_155753-w3wdpl8c/logs/debug-core.log b/reward/wandb/run-20260602_155753-w3wdpl8c/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..9e5b20a931fb5e219e932403dd97ccd0c0fecfe4 --- /dev/null +++ b/reward/wandb/run-20260602_155753-w3wdpl8c/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-06-02T15:57:53.566179759+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpcldbrqe_/port-1119545.txt","pid":1119545,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-02T15:57:53.566743293+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1119545} +{"time":"2026-06-02T15:57:53.566728704+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1119545-1119703-499807096/socket","Net":"unix"}} +{"time":"2026-06-02T15:57:53.747232468+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-02T15:57:53.757340738+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:57:54.66063531+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:57:55.161136168+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"es7pkdemfeik"} +{"time":"2026-06-02T15:57:55.349362363+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"es7pkdemfeik"} +{"time":"2026-06-02T15:57:55.350189216+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:57:55.351867581+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:57:55.351972927+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-02T15:57:55.351994487+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-02T15:57:55.352003686+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-02T15:57:55.352033311+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-02T15:57:55.352102686+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1119545-1119703-499807096/socket","Net":"unix"}} +{"time":"2026-06-02T15:57:55.352194256+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-02T15:57:55.352207188+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-02T15:57:55.352231572+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260602_155753-w3wdpl8c/logs/debug-internal.log b/reward/wandb/run-20260602_155753-w3wdpl8c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1486fc06c3b9ac75c59ede04d9b40269543eb8bc --- /dev/null +++ b/reward/wandb/run-20260602_155753-w3wdpl8c/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-06-02T15:57:53.757501033+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-02T15:57:54.660024712+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-02T15:57:54.660262726+08:00","level":"INFO","msg":"stream: created new stream","id":"w3wdpl8c"} +{"time":"2026-06-02T15:57:54.660359755+08:00","level":"INFO","msg":"handler: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:57:54.660607698+08:00","level":"INFO","msg":"stream: started","id":"w3wdpl8c"} +{"time":"2026-06-02T15:57:54.660630963+08:00","level":"INFO","msg":"writer: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:57:54.660637866+08:00","level":"INFO","msg":"sender: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:57:54.666218691+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-02T15:57:54.920999839+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-06-02T15:57:55.160163367+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":7,"current":11} +{"time":"2026-06-02T15:57:55.294066301+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=beec775a27538504f6741437972a44f7806f1b38def46eca77691a3ae4cfd739"} +{"time":"2026-06-02T15:57:55.294210212+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5df9743fb6d05b18dc36992955ece2ae39cefb60fe66967639d63b0f85d66771"} +{"time":"2026-06-02T15:57:55.29424724+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log18B5349CB0A559E74b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155753-w3wdpl8c/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=beec775a27538504f6741437972a44f7806f1b38def46eca77691a3ae4cfd739, Size: 109}"} +{"time":"2026-06-02T15:57:55.294352982+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml18B5349CB0A57E684b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155753-w3wdpl8c/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5df9743fb6d05b18dc36992955ece2ae39cefb60fe66967639d63b0f85d66771, Size: 5725}"} +{"time":"2026-06-02T15:57:55.295195898+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=714ce9ead81d5b74a717bc4e1a6408f1871297da8cf5505db8031c0ab20fd372"} +{"time":"2026-06-02T15:57:55.295338196+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json18B5349CB0AD2B074b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155753-w3wdpl8c/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=714ce9ead81d5b74a717bc4e1a6408f1871297da8cf5505db8031c0ab20fd372, Size: 989}"} +{"time":"2026-06-02T15:57:55.295572911+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0a57e5c6e401c460aef6a6380d583dc0783d35dcc8c40643bd78b334223ece9e"} +{"time":"2026-06-02T15:57:55.295779999+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json18B5349CB0AFE41A4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155753-w3wdpl8c/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0a57e5c6e401c460aef6a6380d583dc0783d35dcc8c40643bd78b334223ece9e, Size: 326}"} +{"time":"2026-06-02T15:57:55.296810079+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=daddddb2ac7797d4582e291fe08c9ca15435cfb7bd04bfa82ff3c3cc28ed5a11"} +{"time":"2026-06-02T15:57:55.296955387+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt18B5349CB228687A29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155753-w3wdpl8c/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=daddddb2ac7797d4582e291fe08c9ca15435cfb7bd04bfa82ff3c3cc28ed5a11, Size: 2982}"} +{"time":"2026-06-02T15:57:55.304612123+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=af552742e4510ab48af11216f17c2cb5be8a9db51032e35be1236dfd9a940f40"} +{"time":"2026-06-02T15:57:55.304769042+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py18B5349CB03F228817e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155753-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075755Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=af552742e4510ab48af11216f17c2cb5be8a9db51032e35be1236dfd9a940f40, Size: 14790}"} +{"time":"2026-06-02T15:57:55.305725785+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-02T15:57:55.344343894+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-02T15:57:55.350249953+08:00","level":"INFO","msg":"stream: closing","id":"w3wdpl8c"} +{"time":"2026-06-02T15:57:55.350291867+08:00","level":"INFO","msg":"handler: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:57:55.350433413+08:00","level":"INFO","msg":"sender: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:57:55.350453301+08:00","level":"INFO","msg":"stream: closed","id":"w3wdpl8c"} diff --git a/reward/wandb/run-20260602_155753-w3wdpl8c/logs/debug.log b/reward/wandb/run-20260602_155753-w3wdpl8c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ae57c1fb9a41d0a11be46747e052a8976eca6140 --- /dev/null +++ b/reward/wandb/run-20260602_155753-w3wdpl8c/logs/debug.log @@ -0,0 +1,25 @@ +2026-06-02 15:57:53,500 INFO MainThread:1119545 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-02 15:57:53,500 INFO MainThread:1119545 [wandb_setup.py:_flush():81] Configure stats pid to 1119545 +2026-06-02 15:57:53,500 INFO MainThread:1119545 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-02 15:57:53,500 INFO MainThread:1119545 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155753-w3wdpl8c/logs/debug.log +2026-06-02 15:57:53,500 INFO MainThread:1119545 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155753-w3wdpl8c/logs/debug-internal.log +2026-06-02 15:57:53,500 INFO MainThread:1119545 [wandb_init.py:init():844] calling init triggers +2026-06-02 15:57:53,500 INFO MainThread:1119545 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'w3wdpl8c', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_revvideo_orig10k_4gpu_len128', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 10, 'experiment.save_every': 5, 'experiment.eval_every': 5, 'experiment.current_epoch': 7, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/hf/lavida-llada-v1.0-instruct', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_sft_train_cleaned_10k', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.05, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 5e-07, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 5e-07, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-06-02 15:57:53,500 INFO MainThread:1119545 [wandb_init.py:init():892] starting backend +2026-06-02 15:57:53,747 INFO MainThread:1119545 [wandb_init.py:init():895] sending inform_init request +2026-06-02 15:57:53,753 INFO MainThread:1119545 [wandb_init.py:init():903] backend started and connected +2026-06-02 15:57:53,755 INFO MainThread:1119545 [wandb_init.py:init():973] updated telemetry +2026-06-02 15:57:53,760 INFO MainThread:1119545 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-02 15:57:54,914 INFO MainThread:1119545 [wandb_init.py:init():1037] run resumed +2026-06-02 15:57:54,919 INFO MainThread:1119545 [wandb_init.py:init():1042] starting run threads in backend +2026-06-02 15:57:55,019 INFO MainThread:1119545 [wandb_run.py:_console_start():2524] atexit reg +2026-06-02 15:57:55,020 INFO MainThread:1119545 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-02 15:57:55,020 INFO MainThread:1119545 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-02 15:57:55,020 INFO MainThread:1119545 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-02 15:57:55,024 INFO MainThread:1119545 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-02 15:57:55,159 INFO MainThread:1119545 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c +2026-06-02 15:57:55,160 INFO MainThread:1119545 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-02 15:57:55,160 INFO MainThread:1119545 [wandb_run.py:_restore():2472] restore +2026-06-02 15:57:55,160 INFO MainThread:1119545 [wandb_run.py:_restore():2478] restore done +2026-06-02 15:57:55,349 INFO MainThread:1119545 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260602_155753-w3wdpl8c/run-w3wdpl8c.wandb b/reward/wandb/run-20260602_155753-w3wdpl8c/run-w3wdpl8c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ac12bbf31e7070ee2288bc7165c290a7425c8052 Binary files /dev/null and b/reward/wandb/run-20260602_155753-w3wdpl8c/run-w3wdpl8c.wandb differ diff --git a/reward/wandb/run-20260602_155801-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260602_155801-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260602_155801-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260602_155801-w3wdpl8c/files/config.yaml b/reward/wandb/run-20260602_155801-w3wdpl8c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa8e9f528e0c2ff5e447d02e3975d57d4aab80bd --- /dev/null +++ b/reward/wandb/run-20260602_155801-w3wdpl8c/files/config.yaml @@ -0,0 +1,229 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + ktw3zbvie8ipl06ardjn7ktm3qk708cd: + args: + - config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml + - wandb.run_id=w3wdpl8c + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=7 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + host: b1301b38c75f + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-06-02T07:58:01.810743Z" + writerId: ktw3zbvie8ipl06ardjn7ktm3qk708cd + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_sft_train_cleaned_10k +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 7 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_revvideo_orig10k_4gpu_len128/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_revvideo_orig10k_4gpu_len128 +experiment.save_every: + value: 5 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 10 +lr_scheduler.params.learning_rate: + value: 5e-07 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/hf/lavida-llada-v1.0-instruct +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 5e-07 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.05 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: w3wdpl8c diff --git a/reward/wandb/run-20260602_155801-w3wdpl8c/files/output.log b/reward/wandb/run-20260602_155801-w3wdpl8c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6177ff0947e107d2878a505795d61f1ed9fc0911 --- /dev/null +++ b/reward/wandb/run-20260602_155801-w3wdpl8c/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 7 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.0410 avg length: 37.15 diff --git a/reward/wandb/run-20260602_155801-w3wdpl8c/files/requirements.txt b/reward/wandb/run-20260602_155801-w3wdpl8c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260602_155801-w3wdpl8c/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260602_155801-w3wdpl8c/files/wandb-metadata.json b/reward/wandb/run-20260602_155801-w3wdpl8c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3428cfeb1b3f7c02b8d12c09b67f79c0bfe983fd --- /dev/null +++ b/reward/wandb/run-20260602_155801-w3wdpl8c/files/wandb-metadata.json @@ -0,0 +1,26 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-02T07:58:01.810743Z", + "args": [ + "config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml", + "wandb.run_id=w3wdpl8c", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=7" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "writerId": "ktw3zbvie8ipl06ardjn7ktm3qk708cd" +} \ No newline at end of file diff --git a/reward/wandb/run-20260602_155801-w3wdpl8c/files/wandb-summary.json b/reward/wandb/run-20260602_155801-w3wdpl8c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2e35a848bd355ca20270a5e7649f29f4b172b7f3 --- /dev/null +++ b/reward/wandb/run-20260602_155801-w3wdpl8c/files/wandb-summary.json @@ -0,0 +1 @@ +{"_step":10,"reward/unbiased_bonus":0.01875,"_timestamp":1.780382672555827e+09,"reward/train_step":3,"_wandb":{"runtime":3084},"train/loss":0.3788445591926574,"train/learning_rate":8.628481651367888e-08,"_runtime":3084,"reward/avg_response_length":35.435546875,"reward/acc":0.03515625,"reward/t_grpo_bonus":0.3000000000000004} \ No newline at end of file diff --git a/reward/wandb/run-20260602_155801-w3wdpl8c/logs/debug-core.log b/reward/wandb/run-20260602_155801-w3wdpl8c/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..87ea47b722840fc5aa8cbf10b6c18ce48638e2d4 --- /dev/null +++ b/reward/wandb/run-20260602_155801-w3wdpl8c/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-06-02T15:58:01.89350596+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp36qj_rax/port-1119976.txt","pid":1119976,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-02T15:58:01.894466272+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1119976} +{"time":"2026-06-02T15:58:01.894448265+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1119976-1120135-767491754/socket","Net":"unix"}} +{"time":"2026-06-02T15:58:02.066001832+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-02T15:58:02.074426026+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:58:02.886706909+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:58:03.325721339+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ujbebcegarxa"} +{"time":"2026-06-02T15:58:03.538991083+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ujbebcegarxa"} +{"time":"2026-06-02T15:58:03.539666167+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:58:03.540429216+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T15:58:03.540441795+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-02T15:58:03.540452531+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-02T15:58:03.540463894+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-02T15:58:03.540468848+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-02T15:58:03.540558308+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1119976-1120135-767491754/socket","Net":"unix"}} +{"time":"2026-06-02T15:58:03.540581032+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-02T15:58:03.540595549+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-02T15:58:03.540607186+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260602_155801-w3wdpl8c/logs/debug-internal.log b/reward/wandb/run-20260602_155801-w3wdpl8c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..68dbfc0e30e353a3891b72670116afac79736ee2 --- /dev/null +++ b/reward/wandb/run-20260602_155801-w3wdpl8c/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-06-02T15:58:02.07459076+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-02T15:58:02.885954698+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-02T15:58:02.886212384+08:00","level":"INFO","msg":"stream: created new stream","id":"w3wdpl8c"} +{"time":"2026-06-02T15:58:02.886308858+08:00","level":"INFO","msg":"handler: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:58:02.886670942+08:00","level":"INFO","msg":"stream: started","id":"w3wdpl8c"} +{"time":"2026-06-02T15:58:02.886743355+08:00","level":"INFO","msg":"sender: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:58:02.886742664+08:00","level":"INFO","msg":"writer: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:58:02.892960999+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-02T15:58:03.089883927+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-06-02T15:58:03.324579074+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":7,"current":11} +{"time":"2026-06-02T15:58:03.448021152+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ab0513c006c8bbad5d9e0c2de25ecfef2cf13bf3fead00cd9be5ec3026bca5f6"} +{"time":"2026-06-02T15:58:03.448152921+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml18B5349E968993934b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155801-w3wdpl8c/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ab0513c006c8bbad5d9e0c2de25ecfef2cf13bf3fead00cd9be5ec3026bca5f6, Size: 5725}"} +{"time":"2026-06-02T15:58:03.44930542+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ac1edd21df0e2470f5b08bfc8e4b1e60df1242d863501f0b749a05816a106c5c"} +{"time":"2026-06-02T15:58:03.449451472+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json18B5349E9626078B17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155801-w3wdpl8c/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ac1edd21df0e2470f5b08bfc8e4b1e60df1242d863501f0b749a05816a106c5c, Size: 326}"} +{"time":"2026-06-02T15:58:03.449504389+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a6b5600b13eecb32f7ecaf84a99bb9a4a8b5388e2c62adc999b1314e27a417e0"} +{"time":"2026-06-02T15:58:03.449648352+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt18B5349E9633402817e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155801-w3wdpl8c/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a6b5600b13eecb32f7ecaf84a99bb9a4a8b5388e2c62adc999b1314e27a417e0, Size: 2982}"} +{"time":"2026-06-02T15:58:03.495679433+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=29e3ab60ff318b2a0cdbfa9231b6f587a727fb0a2659cbcee39d1199ce541b3d"} +{"time":"2026-06-02T15:58:03.495735021+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=027e14ce10fad69ec8e5025171dac48ea9862b51052c40c43b66f15a6610b964"} +{"time":"2026-06-02T15:58:03.495813916+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log18B5349E960077F217e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155801-w3wdpl8c/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=29e3ab60ff318b2a0cdbfa9231b6f587a727fb0a2659cbcee39d1199ce541b3d, Size: 109}"} +{"time":"2026-06-02T15:58:03.495853901+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json18B5349E967993814b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155801-w3wdpl8c/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=027e14ce10fad69ec8e5025171dac48ea9862b51052c40c43b66f15a6610b964, Size: 989}"} +{"time":"2026-06-02T15:58:03.497394233+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=55c4e4d928b5eecf5bfd6e13798dbb514580126b7559857bcc80571e19e8356b"} +{"time":"2026-06-02T15:58:03.49753674+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py18B5349E98131EFB29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155801-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T075803Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=55c4e4d928b5eecf5bfd6e13798dbb514580126b7559857bcc80571e19e8356b, Size: 14790}"} +{"time":"2026-06-02T15:58:03.497978308+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-02T15:58:03.534587724+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-02T15:58:03.53969659+08:00","level":"INFO","msg":"stream: closing","id":"w3wdpl8c"} +{"time":"2026-06-02T15:58:03.539729986+08:00","level":"INFO","msg":"handler: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:58:03.539814247+08:00","level":"INFO","msg":"sender: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T15:58:03.539830278+08:00","level":"INFO","msg":"stream: closed","id":"w3wdpl8c"} diff --git a/reward/wandb/run-20260602_155801-w3wdpl8c/logs/debug.log b/reward/wandb/run-20260602_155801-w3wdpl8c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2e1f56f84689910eda94cc92f32c6a363d931ffa --- /dev/null +++ b/reward/wandb/run-20260602_155801-w3wdpl8c/logs/debug.log @@ -0,0 +1,25 @@ +2026-06-02 15:58:01,813 INFO MainThread:1119976 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-02 15:58:01,813 INFO MainThread:1119976 [wandb_setup.py:_flush():81] Configure stats pid to 1119976 +2026-06-02 15:58:01,813 INFO MainThread:1119976 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-02 15:58:01,813 INFO MainThread:1119976 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155801-w3wdpl8c/logs/debug.log +2026-06-02 15:58:01,813 INFO MainThread:1119976 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_155801-w3wdpl8c/logs/debug-internal.log +2026-06-02 15:58:01,813 INFO MainThread:1119976 [wandb_init.py:init():844] calling init triggers +2026-06-02 15:58:01,813 INFO MainThread:1119976 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'w3wdpl8c', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_revvideo_orig10k_4gpu_len128', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 10, 'experiment.save_every': 5, 'experiment.eval_every': 5, 'experiment.current_epoch': 7, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/hf/lavida-llada-v1.0-instruct', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_sft_train_cleaned_10k', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.05, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 5e-07, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 5e-07, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-06-02 15:58:01,813 INFO MainThread:1119976 [wandb_init.py:init():892] starting backend +2026-06-02 15:58:02,066 INFO MainThread:1119976 [wandb_init.py:init():895] sending inform_init request +2026-06-02 15:58:02,071 INFO MainThread:1119976 [wandb_init.py:init():903] backend started and connected +2026-06-02 15:58:02,073 INFO MainThread:1119976 [wandb_init.py:init():973] updated telemetry +2026-06-02 15:58:02,079 INFO MainThread:1119976 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-02 15:58:03,084 INFO MainThread:1119976 [wandb_init.py:init():1037] run resumed +2026-06-02 15:58:03,088 INFO MainThread:1119976 [wandb_init.py:init():1042] starting run threads in backend +2026-06-02 15:58:03,185 INFO MainThread:1119976 [wandb_run.py:_console_start():2524] atexit reg +2026-06-02 15:58:03,185 INFO MainThread:1119976 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-02 15:58:03,185 INFO MainThread:1119976 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-02 15:58:03,185 INFO MainThread:1119976 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-02 15:58:03,190 INFO MainThread:1119976 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-02 15:58:03,324 INFO MainThread:1119976 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c +2026-06-02 15:58:03,324 INFO MainThread:1119976 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-02 15:58:03,325 INFO MainThread:1119976 [wandb_run.py:_restore():2472] restore +2026-06-02 15:58:03,325 INFO MainThread:1119976 [wandb_run.py:_restore():2478] restore done +2026-06-02 15:58:03,538 INFO MainThread:1119976 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260602_155801-w3wdpl8c/run-w3wdpl8c.wandb b/reward/wandb/run-20260602_155801-w3wdpl8c/run-w3wdpl8c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..bf73a4adb43ee10a42aef8069b15874b6aef7c4d Binary files /dev/null and b/reward/wandb/run-20260602_155801-w3wdpl8c/run-w3wdpl8c.wandb differ diff --git a/reward/wandb/run-20260602_162447-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260602_162447-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260602_162447-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260602_162447-w3wdpl8c/files/config.yaml b/reward/wandb/run-20260602_162447-w3wdpl8c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd84d6a2ae8c94ccf284e2cb457e85ed57dcf375 --- /dev/null +++ b/reward/wandb/run-20260602_162447-w3wdpl8c/files/config.yaml @@ -0,0 +1,229 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + 0pily6g0mbw3u8mv0c5a10xju7zbz4br: + args: + - config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml + - wandb.run_id=w3wdpl8c + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=8 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + host: b1301b38c75f + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-06-02T08:24:47.874143Z" + writerId: 0pily6g0mbw3u8mv0c5a10xju7zbz4br + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_sft_train_cleaned_10k +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 8 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_revvideo_orig10k_4gpu_len128/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_revvideo_orig10k_4gpu_len128 +experiment.save_every: + value: 5 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 10 +lr_scheduler.params.learning_rate: + value: 5e-07 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/hf/lavida-llada-v1.0-instruct +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 5e-07 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.05 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: w3wdpl8c diff --git a/reward/wandb/run-20260602_162447-w3wdpl8c/files/output.log b/reward/wandb/run-20260602_162447-w3wdpl8c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..72083903875e1a0593d0a80ee8d95a7ec89fe339 --- /dev/null +++ b/reward/wandb/run-20260602_162447-w3wdpl8c/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 8 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.0039 avg length: 37.30 diff --git a/reward/wandb/run-20260602_162447-w3wdpl8c/files/requirements.txt b/reward/wandb/run-20260602_162447-w3wdpl8c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260602_162447-w3wdpl8c/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260602_162447-w3wdpl8c/files/wandb-metadata.json b/reward/wandb/run-20260602_162447-w3wdpl8c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7a6a1163f7c78bc6f4456d724dcb7229f00c69a1 --- /dev/null +++ b/reward/wandb/run-20260602_162447-w3wdpl8c/files/wandb-metadata.json @@ -0,0 +1,26 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-02T08:24:47.874143Z", + "args": [ + "config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml", + "wandb.run_id=w3wdpl8c", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=8" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "writerId": "0pily6g0mbw3u8mv0c5a10xju7zbz4br" +} \ No newline at end of file diff --git a/reward/wandb/run-20260602_162447-w3wdpl8c/files/wandb-summary.json b/reward/wandb/run-20260602_162447-w3wdpl8c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8178742ad5dce0c7cc0851ecfcb1133c612d2133 --- /dev/null +++ b/reward/wandb/run-20260602_162447-w3wdpl8c/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":3689},"train/loss":0.3788445591926574,"train/learning_rate":8.628481651367888e-08,"reward/unbiased_bonus":0.01875,"reward/avg_response_length":35.435546875,"reward/train_step":3,"_timestamp":1.780382672555827e+09,"reward/t_grpo_bonus":0.3000000000000004,"_step":10,"reward/acc":0.03515625,"_runtime":3689} \ No newline at end of file diff --git a/reward/wandb/run-20260602_162447-w3wdpl8c/logs/debug-core.log b/reward/wandb/run-20260602_162447-w3wdpl8c/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..f42b6d4a5878a93369ee503867e6712a2f090e4e --- /dev/null +++ b/reward/wandb/run-20260602_162447-w3wdpl8c/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-06-02T16:24:47.955390517+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmprr5lfwah/port-1174257.txt","pid":1174257,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-02T16:24:47.956423504+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1174257} +{"time":"2026-06-02T16:24:47.956387622+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1174257-1174421-934569183/socket","Net":"unix"}} +{"time":"2026-06-02T16:24:48.129627509+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-02T16:24:48.139539634+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T16:24:48.557984399+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T16:24:49.032782966+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4w4mgxyrw7yy"} +{"time":"2026-06-02T16:24:49.251524379+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4w4mgxyrw7yy"} +{"time":"2026-06-02T16:24:49.25310718+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T16:24:49.254017078+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T16:24:49.254081543+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-02T16:24:49.254100098+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-02T16:24:49.254106215+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-02T16:24:49.254117649+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-02T16:24:49.254202561+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1174257-1174421-934569183/socket","Net":"unix"}} +{"time":"2026-06-02T16:24:49.254231276+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-02T16:24:49.254251392+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-02T16:24:49.254268142+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260602_162447-w3wdpl8c/logs/debug-internal.log b/reward/wandb/run-20260602_162447-w3wdpl8c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9b195ccd976977d968e90064e2abc89d49df058f --- /dev/null +++ b/reward/wandb/run-20260602_162447-w3wdpl8c/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-06-02T16:24:48.139793091+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-02T16:24:48.55737745+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-02T16:24:48.55760008+08:00","level":"INFO","msg":"stream: created new stream","id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:48.557697909+08:00","level":"INFO","msg":"handler: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:48.557952239+08:00","level":"INFO","msg":"stream: started","id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:48.55798561+08:00","level":"INFO","msg":"sender: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:48.558029251+08:00","level":"INFO","msg":"writer: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:48.563275286+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-02T16:24:48.801616695+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-06-02T16:24:49.031581239+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":8,"current":11} +{"time":"2026-06-02T16:24:49.164487947+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=93fff1b47beabbd146457fb5bd3ac8ef483ae7534e12cd5b46afaac3f25e1210"} +{"time":"2026-06-02T16:24:49.164879275+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log18B5361472BB2ECD4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162447-w3wdpl8c/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=93fff1b47beabbd146457fb5bd3ac8ef483ae7534e12cd5b46afaac3f25e1210, Size: 109}"} +{"time":"2026-06-02T16:24:49.166137519+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=926d147222e8c09f8c1f3b9363b333be53b33e98cf50e81b3223008b18fa7e5e"} +{"time":"2026-06-02T16:24:49.166262041+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json18B536147259842117e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162447-w3wdpl8c/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=926d147222e8c09f8c1f3b9363b333be53b33e98cf50e81b3223008b18fa7e5e, Size: 989}"} +{"time":"2026-06-02T16:24:49.167029455+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=805b14757a90d70422d182831c30d315878795412fbcca88d14e6d36fc1d3374"} +{"time":"2026-06-02T16:24:49.167169543+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt18B53614745B2F4F29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162447-w3wdpl8c/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=805b14757a90d70422d182831c30d315878795412fbcca88d14e6d36fc1d3374, Size: 2982}"} +{"time":"2026-06-02T16:24:49.16880386+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=83caf8bb620dbd9a0218fe5b148ea9224d51a6854a620628b9cff2427e13c6ed"} +{"time":"2026-06-02T16:24:49.168961308+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml18B53614727822BF17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162447-w3wdpl8c/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=83caf8bb620dbd9a0218fe5b148ea9224d51a6854a620628b9cff2427e13c6ed, Size: 5725}"} +{"time":"2026-06-02T16:24:49.178424216+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7b99c1d1cbffe48d1b433bac4b83aa2f0f338cc43390eb150169de5bd1bd430c"} +{"time":"2026-06-02T16:24:49.178570009+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py18B53614730035974b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162447-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7b99c1d1cbffe48d1b433bac4b83aa2f0f338cc43390eb150169de5bd1bd430c, Size: 14790}"} +{"time":"2026-06-02T16:24:49.205225392+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e0c75115dadef97736dadc7418c53ec54ef2246927a2369705f4f38cfc284b02"} +{"time":"2026-06-02T16:24:49.20541017+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json18B536147428B5B729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162447-w3wdpl8c/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082449Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e0c75115dadef97736dadc7418c53ec54ef2246927a2369705f4f38cfc284b02, Size: 326}"} +{"time":"2026-06-02T16:24:49.206124033+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-02T16:24:49.248590974+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-02T16:24:49.253156057+08:00","level":"INFO","msg":"stream: closing","id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:49.253194854+08:00","level":"INFO","msg":"handler: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:49.253289362+08:00","level":"INFO","msg":"sender: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:49.253300718+08:00","level":"INFO","msg":"stream: closed","id":"w3wdpl8c"} diff --git a/reward/wandb/run-20260602_162447-w3wdpl8c/logs/debug.log b/reward/wandb/run-20260602_162447-w3wdpl8c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3724e51d4d517781580b24ae54b6e6eee1d8bf52 --- /dev/null +++ b/reward/wandb/run-20260602_162447-w3wdpl8c/logs/debug.log @@ -0,0 +1,25 @@ +2026-06-02 16:24:47,876 INFO MainThread:1174257 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-02 16:24:47,876 INFO MainThread:1174257 [wandb_setup.py:_flush():81] Configure stats pid to 1174257 +2026-06-02 16:24:47,876 INFO MainThread:1174257 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-02 16:24:47,877 INFO MainThread:1174257 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162447-w3wdpl8c/logs/debug.log +2026-06-02 16:24:47,877 INFO MainThread:1174257 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162447-w3wdpl8c/logs/debug-internal.log +2026-06-02 16:24:47,877 INFO MainThread:1174257 [wandb_init.py:init():844] calling init triggers +2026-06-02 16:24:47,877 INFO MainThread:1174257 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'w3wdpl8c', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_revvideo_orig10k_4gpu_len128', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 10, 'experiment.save_every': 5, 'experiment.eval_every': 5, 'experiment.current_epoch': 8, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/hf/lavida-llada-v1.0-instruct', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_sft_train_cleaned_10k', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.05, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 5e-07, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 5e-07, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-06-02 16:24:47,877 INFO MainThread:1174257 [wandb_init.py:init():892] starting backend +2026-06-02 16:24:48,130 INFO MainThread:1174257 [wandb_init.py:init():895] sending inform_init request +2026-06-02 16:24:48,136 INFO MainThread:1174257 [wandb_init.py:init():903] backend started and connected +2026-06-02 16:24:48,138 INFO MainThread:1174257 [wandb_init.py:init():973] updated telemetry +2026-06-02 16:24:48,142 INFO MainThread:1174257 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-02 16:24:48,795 INFO MainThread:1174257 [wandb_init.py:init():1037] run resumed +2026-06-02 16:24:48,799 INFO MainThread:1174257 [wandb_init.py:init():1042] starting run threads in backend +2026-06-02 16:24:48,895 INFO MainThread:1174257 [wandb_run.py:_console_start():2524] atexit reg +2026-06-02 16:24:48,895 INFO MainThread:1174257 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-02 16:24:48,895 INFO MainThread:1174257 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-02 16:24:48,895 INFO MainThread:1174257 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-02 16:24:48,899 INFO MainThread:1174257 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-02 16:24:49,031 INFO MainThread:1174257 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c +2026-06-02 16:24:49,031 INFO MainThread:1174257 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-02 16:24:49,032 INFO MainThread:1174257 [wandb_run.py:_restore():2472] restore +2026-06-02 16:24:49,032 INFO MainThread:1174257 [wandb_run.py:_restore():2478] restore done +2026-06-02 16:24:49,252 INFO MainThread:1174257 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260602_162447-w3wdpl8c/run-w3wdpl8c.wandb b/reward/wandb/run-20260602_162447-w3wdpl8c/run-w3wdpl8c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0c27b2c5c124027528a18c7be56f9f4e7bf785e3 Binary files /dev/null and b/reward/wandb/run-20260602_162447-w3wdpl8c/run-w3wdpl8c.wandb differ diff --git a/reward/wandb/run-20260602_162455-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260602_162455-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260602_162455-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260602_162455-w3wdpl8c/files/config.yaml b/reward/wandb/run-20260602_162455-w3wdpl8c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e0f27dd75c7d1fedead214b37a18538b6094fa7 --- /dev/null +++ b/reward/wandb/run-20260602_162455-w3wdpl8c/files/config.yaml @@ -0,0 +1,229 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + 5or8kxewit81il7wz7pasqef71e961dr: + args: + - config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml + - wandb.run_id=w3wdpl8c + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=8 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + host: b1301b38c75f + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-06-02T08:24:55.732140Z" + writerId: 5or8kxewit81il7wz7pasqef71e961dr + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_sft_train_cleaned_10k +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 8 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_revvideo_orig10k_4gpu_len128/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_revvideo_orig10k_4gpu_len128 +experiment.save_every: + value: 5 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 10 +lr_scheduler.params.learning_rate: + value: 5e-07 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/hf/lavida-llada-v1.0-instruct +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 5e-07 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.05 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: w3wdpl8c diff --git a/reward/wandb/run-20260602_162455-w3wdpl8c/files/output.log b/reward/wandb/run-20260602_162455-w3wdpl8c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..72083903875e1a0593d0a80ee8d95a7ec89fe339 --- /dev/null +++ b/reward/wandb/run-20260602_162455-w3wdpl8c/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 8 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.0039 avg length: 37.30 diff --git a/reward/wandb/run-20260602_162455-w3wdpl8c/files/requirements.txt b/reward/wandb/run-20260602_162455-w3wdpl8c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260602_162455-w3wdpl8c/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260602_162455-w3wdpl8c/files/wandb-metadata.json b/reward/wandb/run-20260602_162455-w3wdpl8c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ba89949e087efb9711bf5370c6d80be56cd13719 --- /dev/null +++ b/reward/wandb/run-20260602_162455-w3wdpl8c/files/wandb-metadata.json @@ -0,0 +1,26 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-02T08:24:55.732140Z", + "args": [ + "config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml", + "wandb.run_id=w3wdpl8c", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=8" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "writerId": "5or8kxewit81il7wz7pasqef71e961dr" +} \ No newline at end of file diff --git a/reward/wandb/run-20260602_162455-w3wdpl8c/files/wandb-summary.json b/reward/wandb/run-20260602_162455-w3wdpl8c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5bfe1c6f9949500698d22dd42fb2d6178d4ae21a --- /dev/null +++ b/reward/wandb/run-20260602_162455-w3wdpl8c/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/learning_rate":8.628481651367888e-08,"train/loss":0.3788445591926574,"_step":10,"reward/unbiased_bonus":0.01875,"_runtime":3689,"_timestamp":1.780382672555827e+09,"reward/train_step":3,"reward/acc":0.03515625,"_wandb":{"runtime":3689},"reward/t_grpo_bonus":0.3000000000000004,"reward/avg_response_length":35.435546875} \ No newline at end of file diff --git a/reward/wandb/run-20260602_162455-w3wdpl8c/logs/debug-core.log b/reward/wandb/run-20260602_162455-w3wdpl8c/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..486a833d0dff409e1841d6daf185e59446a65738 --- /dev/null +++ b/reward/wandb/run-20260602_162455-w3wdpl8c/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-06-02T16:24:55.815618866+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp5xlawav9/port-1174680.txt","pid":1174680,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-02T16:24:55.816630428+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1174680} +{"time":"2026-06-02T16:24:55.816558994+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1174680-1174844-1131600006/socket","Net":"unix"}} +{"time":"2026-06-02T16:24:55.987571579+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-02T16:24:55.997997165+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T16:24:56.613297017+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T16:24:57.050464562+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"m5lr1gdq8c3c"} +{"time":"2026-06-02T16:24:57.246414871+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"m5lr1gdq8c3c"} +{"time":"2026-06-02T16:24:57.247288166+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T16:24:57.248105905+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T16:24:57.24876522+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-02T16:24:57.248824321+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-02T16:24:57.248834421+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-02T16:24:57.248916153+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-02T16:24:57.248928901+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-02T16:24:57.248849284+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-02T16:24:57.249070079+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1174680-1174844-1131600006/socket","Net":"unix"}} +{"time":"2026-06-02T16:24:57.249121063+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260602_162455-w3wdpl8c/logs/debug-internal.log b/reward/wandb/run-20260602_162455-w3wdpl8c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e8652a519a0ca944a0ccdda97bf045ff9d985ed2 --- /dev/null +++ b/reward/wandb/run-20260602_162455-w3wdpl8c/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-06-02T16:24:55.998325686+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-02T16:24:56.612700715+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-02T16:24:56.61294248+08:00","level":"INFO","msg":"stream: created new stream","id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:56.613056147+08:00","level":"INFO","msg":"handler: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:56.613264862+08:00","level":"INFO","msg":"stream: started","id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:56.613321999+08:00","level":"INFO","msg":"writer: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:56.61332115+08:00","level":"INFO","msg":"sender: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:56.616248572+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-02T16:24:56.812228735+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-06-02T16:24:57.048653618+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":8,"current":11} +{"time":"2026-06-02T16:24:57.184441142+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5e1400291ddb34737dd0b077f5c26f4d6f37f7588d25c0d230e6e375965ae1e8"} +{"time":"2026-06-02T16:24:57.184607928+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml18B53616504F7D6017e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162455-w3wdpl8c/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5e1400291ddb34737dd0b077f5c26f4d6f37f7588d25c0d230e6e375965ae1e8, Size: 5725}"} +{"time":"2026-06-02T16:24:57.184828296+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b83dcf11b92c8dad7b3b5121cb2c58dcf75067ff03f356f154f801c8a93c804b"} +{"time":"2026-06-02T16:24:57.184931207+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json18B53616523DC37829678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162455-w3wdpl8c/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=b83dcf11b92c8dad7b3b5121cb2c58dcf75067ff03f356f154f801c8a93c804b, Size: 326}"} +{"time":"2026-06-02T16:24:57.188513359+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=dc27496fb65e7b0e9054ffaa890ad92d879b90acbf7b745610e3b12557137614"} +{"time":"2026-06-02T16:24:57.188691357+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json18B536165079451317e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162455-w3wdpl8c/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=dc27496fb65e7b0e9054ffaa890ad92d879b90acbf7b745610e3b12557137614, Size: 989}"} +{"time":"2026-06-02T16:24:57.190243685+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1e75642294d28fefac697381e80fbc109f8e26a36185b29a4ae40f6325d4d6b7"} +{"time":"2026-06-02T16:24:57.190417186+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt18B53616526C97C529678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162455-w3wdpl8c/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1e75642294d28fefac697381e80fbc109f8e26a36185b29a4ae40f6325d4d6b7, Size: 2982}"} +{"time":"2026-06-02T16:24:57.191847525+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1c1176c999655db398438fce5e6279f93fccc205c12538516520da9dd152100d"} +{"time":"2026-06-02T16:24:57.191959568+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py18B536165236A82B29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162455-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1c1176c999655db398438fce5e6279f93fccc205c12538516520da9dd152100d, Size: 14790}"} +{"time":"2026-06-02T16:24:57.193091532+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0bb1801088ca86df7ebd09156cbb5646e443cf7cbf403956b4f6256d0d3f679b"} +{"time":"2026-06-02T16:24:57.193241121+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log18B5361650BEDDBE17e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162455-w3wdpl8c/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T082457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0bb1801088ca86df7ebd09156cbb5646e443cf7cbf403956b4f6256d0d3f679b, Size: 109}"} +{"time":"2026-06-02T16:24:57.193531182+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-02T16:24:57.236301195+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-02T16:24:57.247338874+08:00","level":"INFO","msg":"stream: closing","id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:57.247373161+08:00","level":"INFO","msg":"handler: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:57.247499074+08:00","level":"INFO","msg":"sender: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T16:24:57.247515025+08:00","level":"INFO","msg":"stream: closed","id":"w3wdpl8c"} diff --git a/reward/wandb/run-20260602_162455-w3wdpl8c/logs/debug.log b/reward/wandb/run-20260602_162455-w3wdpl8c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..aed2b6d5d6163bf01c76a19cb242c74e00d359d2 --- /dev/null +++ b/reward/wandb/run-20260602_162455-w3wdpl8c/logs/debug.log @@ -0,0 +1,25 @@ +2026-06-02 16:24:55,735 INFO MainThread:1174680 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-02 16:24:55,735 INFO MainThread:1174680 [wandb_setup.py:_flush():81] Configure stats pid to 1174680 +2026-06-02 16:24:55,735 INFO MainThread:1174680 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-02 16:24:55,735 INFO MainThread:1174680 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162455-w3wdpl8c/logs/debug.log +2026-06-02 16:24:55,735 INFO MainThread:1174680 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_162455-w3wdpl8c/logs/debug-internal.log +2026-06-02 16:24:55,735 INFO MainThread:1174680 [wandb_init.py:init():844] calling init triggers +2026-06-02 16:24:55,735 INFO MainThread:1174680 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'w3wdpl8c', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_revvideo_orig10k_4gpu_len128', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 10, 'experiment.save_every': 5, 'experiment.eval_every': 5, 'experiment.current_epoch': 8, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/hf/lavida-llada-v1.0-instruct', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_sft_train_cleaned_10k', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.05, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 5e-07, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 5e-07, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-06-02 16:24:55,735 INFO MainThread:1174680 [wandb_init.py:init():892] starting backend +2026-06-02 16:24:55,987 INFO MainThread:1174680 [wandb_init.py:init():895] sending inform_init request +2026-06-02 16:24:55,993 INFO MainThread:1174680 [wandb_init.py:init():903] backend started and connected +2026-06-02 16:24:55,995 INFO MainThread:1174680 [wandb_init.py:init():973] updated telemetry +2026-06-02 16:24:56,001 INFO MainThread:1174680 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-02 16:24:56,807 INFO MainThread:1174680 [wandb_init.py:init():1037] run resumed +2026-06-02 16:24:56,810 INFO MainThread:1174680 [wandb_init.py:init():1042] starting run threads in backend +2026-06-02 16:24:56,906 INFO MainThread:1174680 [wandb_run.py:_console_start():2524] atexit reg +2026-06-02 16:24:56,906 INFO MainThread:1174680 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-02 16:24:56,906 INFO MainThread:1174680 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-02 16:24:56,906 INFO MainThread:1174680 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-02 16:24:56,910 INFO MainThread:1174680 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-02 16:24:57,048 INFO MainThread:1174680 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c +2026-06-02 16:24:57,049 INFO MainThread:1174680 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-02 16:24:57,049 INFO MainThread:1174680 [wandb_run.py:_restore():2472] restore +2026-06-02 16:24:57,049 INFO MainThread:1174680 [wandb_run.py:_restore():2478] restore done +2026-06-02 16:24:57,246 INFO MainThread:1174680 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260602_162455-w3wdpl8c/run-w3wdpl8c.wandb b/reward/wandb/run-20260602_162455-w3wdpl8c/run-w3wdpl8c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..336135ab6a516b413968e6150898b4b6a79937ac Binary files /dev/null and b/reward/wandb/run-20260602_162455-w3wdpl8c/run-w3wdpl8c.wandb differ diff --git a/reward/wandb/run-20260602_172441-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py b/reward/wandb/run-20260602_172441-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c1b394a00b9fb1c9602a9d51cf65032e639ae --- /dev/null +++ b/reward/wandb/run-20260602_172441-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py @@ -0,0 +1,339 @@ +import json +import math_utils_v +import nest_asyncio +from concurrent.futures import ThreadPoolExecutor +import asyncio +import re +from termcolor import cprint +from omegaconf import MISSING +from omegaconf import DictConfig, ListConfig, OmegaConf +from typing import List, Union +import os +from collections import defaultdict +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +import wandb +from train.utils import flatten_omega_conf, ensure_wandb_run_id + +def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None: + p = sample.get(f"{media_type}_abs_path") + if p is None: + p = sample.get(media_type) or sample.get(f"{media_type}_path") + if p is None: + return None + if not os.path.isabs(p) and dataset_root: + p = os.path.join(dataset_root, p) + try: + return os.path.abspath(p) + except Exception: + return p + +def get_config(): + cli_conf = OmegaConf.from_cli() + yaml_conf = OmegaConf.load(cli_conf.config) + conf = OmegaConf.merge(yaml_conf, cli_conf) + return conf + +def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]: + token = '<|endoftext|>' + if isinstance(obj, str): + return obj.replace(token, '') + if isinstance(obj, list): + return [s.replace(token, '') if isinstance(s, str) else s for s in obj] + raise TypeError("Expected str or list of str") + + +if __name__ == "__main__": + + config = get_config() + run = None + if config.experiment.function == "train": + existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None) + run_id = ensure_wandb_run_id(config) + resume_wandb_run = config.wandb.resume + if existing_run_id is None: + resume_wandb_run = False + wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)} + run = wandb.init( + project=config.experiment.project, + entity=config.wandb.get("entity", None), + id=run_id, + resume=resume_wandb_run, + config=wandb_config, + reinit=True, + name=config.experiment.project, + ) + + project_name = config.experiment.project + + if config.experiment.current_epoch == 1: + pretrained_model = config.model.pretrained_model + else: + pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name + + if config.experiment.function == "train": + shrink = config.training.shrink + dataset = config.dataset.train_dataset + outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + elif config.experiment.function == "evaluation": + dataset = config.evaluation.eval_dataset + outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}" + + file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json" + + with open(file_name, 'r') as f: + data = json.load(f) + + index_list = [] + extracted_output_list = [] + response_length_list = [] + for i in range(len(data)): + data[i]["correctness"] = [] + response_length_list += data[i]["response_length"] + index_list += [i] * len(data[i]["extracted_output"]) + extracted_output_list += remove_endoftext(data[i]["extracted_output"]) + + nest_asyncio.apply() + + async def get_correctness_all(): + executor = ThreadPoolExecutor(max_workers=64) + tasks = [] + + for p in range(len(index_list)): + pred = extracted_output_list[p] + idx = index_list[p] + + # 选择题正确性匹配逻辑 + if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse": + gt = data[idx].get("response", "") + + async def mcq_check(p_str, g_str): + if p_str in g_str or g_str in p_str: + return True + + p_str = str(p_str).strip() + g_str = str(g_str).strip() + p_match = re.search(r'^([A-Z])[\.\s]', p_str) + g_match = re.search(r'^([A-Z])[\.\s]', g_str) + if p_match and g_match: + return p_match.group(1) == g_match.group(1) + return g_str in p_str + + tasks.append(mcq_check(pred, gt)) + else: + gt = data[idx].get("ground_truth_answer", "") + tasks.append(math_utils_v.is_equal(pred, gt, executor)) + + return await asyncio.gather(*tasks) + + correctness_all = asyncio.run(get_correctness_all()) + for p, ok in enumerate(correctness_all): + index_i = index_list[p] + data[index_i]["correctness"].append(bool(ok)) + + # ========================================================================= + # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励 + # ========================================================================= + + # 1. 按照 data_id 分组映射索引 + grouped_data = defaultdict(lambda: { + "forward_False": [], "backward_False": [], + "forward_True": [], "backward_True": [] + }) + + for i in range(len(data)): + d_id = data[i].get("data_id") + if d_id is None: + continue + r_type = data[i].get("reversal_type", "forward") + is_shuf = str(data[i].get("is_shuffled", False)) + grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i) + + # 计算并应用群组奖励 + all_temporal_bonuses = [] # 新增：记录所有的时序红利 + all_unbiased_bonuses = [] # 新增：记录所有的无偏红利 + + # 2. 计算并应用群组奖励 + for d_id, group in grouped_data.items(): + + def get_acc(key): + if not group[key]: return 0.0 + idx = group[key][0] + corr = data[idx]["correctness"] + return sum(corr) / len(corr) if len(corr) > 0 else 0.0 + + # 获取四组情况的正确率 + acc_f_ord = get_acc("forward_False") + acc_f_shuf = get_acc("forward_True") + acc_b_ord = get_acc("backward_False") + acc_b_shuf = get_acc("backward_True") + + # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3 + f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0 + b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0 + + # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3 + unbiased_bonus = 0.0 + if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1): + unbiased_bonus = 0.3 + + # 新增：将当前组的奖励记录下来用于统计 + if group["forward_False"]: + all_temporal_bonuses.append(f_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + if group["backward_False"]: + all_temporal_bonuses.append(b_temporal_bonus) + all_unbiased_bonuses.append(unbiased_bonus) + + # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本 + def apply_bonus(key, temporal_bonus, unbiased_bonus): + if not group[key]: return + idx = group[key][0] + raw_rewards = [] + for is_correct in data[idx]["correctness"]: + if is_correct: + # 基础分 1.0 + 时序红利 + 无偏红利 + raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus) + else: + raw_rewards.append(0.0) + data[idx]["raw_rewards"] = raw_rewards + + apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus) + apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus) + + # ========================================================================= + + def z_score_normalize(lst): + mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0 + std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0 + if std == 0: + return [0 for _ in lst] + return [(x - mean) / std for x in lst] + + final_data = [] + ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据 + + for i in range(len(data)): + # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习 + if data[i].get("is_shuffled", False): + continue + + ordered_correctness.extend(data[i]["correctness"]) + + # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数 + raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]]) + lengths = data[i]["response_length"] + + # 长度惩罚 + if config.reward.strict_len_check: + for j in range(len(lengths)): + if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5: + raw_rewards[j] = 0.0 + if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5: + raw_rewards[j] = 0.0 + + # Z-Score 优势归一化 + rewards = z_score_normalize(raw_rewards) + data[i]["rewards"] = rewards + + if config.experiment.function == "train": + proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0 + # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本 + if proportion > 0.95 or proportion < 0.05: + continue + + for j in range(len(rewards)): + data_i = {} + data_i["prompt"] = data[i]["prompt"] + data_i["reward"] = rewards[j] + data_i["response"] = data[i]["full_output"][j] + data_i["step_map"] = data[i]["step_map"][j] + + # 保留其他需要的上下文 ID + data_i["data_id"] = data[i].get("data_id") + data_i["reversal_type"] = data[i].get("reversal_type") + + if "image_token_ids" in data[i]: + data_i["image_token_ids"] = data[i]["image_token_ids"] + + p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video") + if p_video_abs is not None: + data_i["video_abs_path"] = p_video_abs + + p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image") + if p_image_abs is not None: + data_i["image_abs_path"] = p_image_abs + + final_data.append(data_i) + + if config.experiment.function == "evaluation": + data[i]["step_map"] = [] + + if config.experiment.function == "train": + with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f: + json.dump(final_data, f, indent=2, ensure_ascii=False) + + + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + + outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt" + os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True) + with open(outputs_result_name, "a") as f: + # Save + print + def save_and_print(text): + cprint("\n\n\n" + text, color="green") + f.write(text + "\n") + + # 只计算有序组（有效学习组）的准确率来作为监控指标 + acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan") + avg_len = sum(response_length_list)/len(response_length_list) + + # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值 + avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0 + avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0 + + output_text = f"train step: {config.experiment.current_epoch} " + if config.experiment.function == "train": + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} block_size: {config.rollout.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy} top_k: {config.rollout.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + if config.model.model_base != "sdar" and config.model.model_base != "trado": + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} block_size: {config.evaluation.block_size} acc: {acc:.4f} avg length: {avg_len:.2f}" + else: + output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy} top_k: {config.evaluation.top_k} acc: {acc:.4f} avg length: {avg_len:.2f}" + save_and_print(output_text) + + if config.experiment.function == "train": + csv_file_path = f"../{project_name}/results/all_rewards.csv" + + # 如果是第1个 epoch/step，或者是新文件，就先写入表头 + write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1 + + with open(csv_file_path, "a", encoding="utf-8") as f_csv: + if write_header: + f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n") + + # 按行追加当前 step 的数据 + f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n") + + if run is not None: + wandb.log( + { + "reward/acc": acc, + "reward/t_grpo_bonus": avg_temp, + "reward/unbiased_bonus": avg_unbias, + "reward/avg_response_length": avg_len, + "reward/train_step": config.experiment.current_epoch, + }, + step=config.experiment.current_epoch, + ) + wandb.finish() diff --git a/reward/wandb/run-20260602_172441-w3wdpl8c/files/config.yaml b/reward/wandb/run-20260602_172441-w3wdpl8c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db146a71a6f9176b1eb301ca97a864b81f4835ab --- /dev/null +++ b/reward/wandb/run-20260602_172441-w3wdpl8c/files/config.yaml @@ -0,0 +1,229 @@ +_wandb: + value: + cli_version: 0.25.0 + code_path: code/reward/rl_reward_video_reversal.py + e: + xvb1lk1cnls4lggnap8yupkndjuv02j1: + args: + - config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml + - wandb.run_id=w3wdpl8c + - experiment.function=train + - evaluation.block_size=None + - evaluation.top_k=None + - evaluation.remasking_strategy=None + - experiment.current_epoch=10 + codePath: reward/rl_reward_video_reversal.py + codePathLocal: rl_reward_video_reversal.py + email: shixi_runqing@staff.weibo.com + executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python + git: + commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255 + remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git + host: b1301b38c75f + os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35 + program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py + python: CPython 3.10.19 + root: /data4/runqing/dLLM-RL/reward + startedAt: "2026-06-02T09:24:41.968601Z" + writerId: xvb1lk1cnls4lggnap8yupkndjuv02j1 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 41 + "2": + - 1 + - 41 + "3": + - 2 + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.10.19 + "5": 0.25.0 + "10": + - 20 + "12": 0.25.0 + "13": linux-x86_64 +config: + value: ../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml +dataset.data_type: + value: video_reverse +dataset.optimization_data: + value: rl_data +dataset.train_dataset: + value: video_reversal_sft_train_cleaned_10k +evaluation.batch_size: + value: 1 +evaluation.block_size: + value: None +evaluation.data_type: + value: video_reverse +evaluation.dynamic_threshold: + value: 0.9 +evaluation.eval_dataset: + value: video_reversal_rl_eval +evaluation.further_horizon: + value: 128 +evaluation.if_eval: + value: false +evaluation.max_gen_length: + value: 128 +evaluation.num_response_per_task: + value: 3 +evaluation.remasking_strategy: + value: None +evaluation.remasking_strategy.0: + value: low_confidence_static +evaluation.remasking_strategy.1: + value: low_confidence_dynamic +evaluation.steps: + value: 128 +evaluation.target: + value: confidence +evaluation.temperature: + value: 0.1 +evaluation.top_k: + value: None +evaluation.use_cache: + value: true +execute.num_chunk: + value: 128 +experiment.current_epoch: + value: 10 +experiment.deepspeed_file: + value: 1_node_8_gpus_deepspeed_zero2 +experiment.eval_every: + value: 5 +experiment.function: + value: train +experiment.logging_dir: + value: rl_lavida_revvideo_orig10k_4gpu_len128/logs +experiment.node_index: + value: 0 +experiment.num_node: + value: 1 +experiment.project: + value: rl_lavida_revvideo_orig10k_4gpu_len128 +experiment.save_every: + value: 5 +experiment.start_from_scratch: + value: true +experiment.total_step: + value: 10 +lr_scheduler.params.learning_rate: + value: 5e-07 +lr_scheduler.params.min_lr_scale: + value: 1 +lr_scheduler.params.warmup_steps: + value: 0 +lr_scheduler.scheduler: + value: cosine +model.image_resolution: + value: 512 +model.model_base: + value: lavida +model.optimized_name: + value: optimized +model.pretrained_model: + value: /data4/runqing/hf/lavida-llada-v1.0-instruct +model.vq_model_path: + value: /data4/runqing/hf/magvitv2 +optimizer.name: + value: adamw +optimizer.params.beta1: + value: 0.9 +optimizer.params.beta2: + value: 0.999 +optimizer.params.epsilon: + value: 1e-08 +optimizer.params.learning_rate: + value: 5e-07 +optimizer.params.scale_lr: + value: false +optimizer.params.weight_decay: + value: 0 +reward.answer_must_in_box: + value: true +reward.strict_len_check: + value: false +rollout.batch_size: + value: 1 +rollout.block_size: + value: 32 +rollout.dynamic_threshold: + value: 0.9 +rollout.further_horizon: + value: 128 +rollout.max_gen_length: + value: 128 +rollout.num_response_per_task: + value: 8 +rollout.num_task_per_step: + value: 128 +rollout.remasking_strategy: + value: low_confidence_dynamic +rollout.steps: + value: 128 +rollout.target: + value: confidence +rollout.temperature: + value: 0.8 +rollout.use_cache: + value: true +system.HF_HOME: + value: null +system.HTTP_PROXY: + value: null +system.base_dir: + value: /data4/runqing/dLLM-RL +system.env_name: + value: /data8/runqing/miniconda/envs/dllm-rl +system.envs_dir: + value: /data8/runqing/miniconda/envs/dllm-rl +training.batch_size_lm: + value: 1 +training.beta: + value: 0.05 +training.enable_tf32: + value: true +training.eps: + value: 0.2 +training.gradient_accumulation_steps: + value: 2 +training.lower_p: + value: 0.35 +training.mask_times_per_sample: + value: 2 +training.max_gen_length: + value: 128 +training.max_grad_norm: + value: 1 +training.max_prompt_len: + value: 512 +training.method: + value: random_masking +training.mixed_precision: + value: bf16 +training.num_train_epochs: + value: 1 +training.post_num: + value: 1 +training.seed: + value: 10086 +training.shrink: + value: 8 +training.upper_p: + value: 0.9 +training.use_kl_estimator_k3: + value: true +wandb.entity: + value: null +wandb.resume: + value: auto +wandb.run_id: + value: w3wdpl8c diff --git a/reward/wandb/run-20260602_172441-w3wdpl8c/files/output.log b/reward/wandb/run-20260602_172441-w3wdpl8c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..eeffd3fff7dfce4ee88eff6d13e7352b68db27bf --- /dev/null +++ b/reward/wandb/run-20260602_172441-w3wdpl8c/files/output.log @@ -0,0 +1,4 @@ + + + +train step: 10 remasking_strategy: low_confidence_dynamic block_size: 32 acc: 0.0234 avg length: 41.87 diff --git a/reward/wandb/run-20260602_172441-w3wdpl8c/files/requirements.txt b/reward/wandb/run-20260602_172441-w3wdpl8c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..72500b9ab07fa4e88881df34659dae386e10da72 --- /dev/null +++ b/reward/wandb/run-20260602_172441-w3wdpl8c/files/requirements.txt @@ -0,0 +1,157 @@ +transformers==4.52.4 +nvidia-nvtx-cu12==12.4.127 +deepspeed==0.18.6 +typing-inspection==0.4.2 +scipy==1.15.3 +MarkupSafe==3.0.3 +nvidia-cuda-runtime-cu12==12.4.127 +detoxify==0.5.2 +yarl==1.22.0 +tornado==6.5.4 +nvidia-curand-cu12==10.3.5.147 +nest-asyncio==1.6.0 +safetensors==0.7.0 +async-timeout==5.0.1 +typing_extensions==4.15.0 +asttokens==3.0.1 +cuda-python==13.1.1 +executing==2.2.1 +nvidia-cudnn-frontend==1.18.0 +comm==0.2.3 +tokenizers==0.21.4 +six==1.17.0 +Pygments==2.19.2 +nvidia-cusparselt-cu12==0.6.2 +apache-tvm-ffi==0.1.8.post2 +flash_attn==2.7.4.post1 +tqdm==4.67.3 +tensorboard-data-server==0.7.2 +annotated-types==0.7.0 +math-verify==0.9.0 +idna==3.11 +regex==2026.1.15 +nvidia-cutlass-dsl-libs-base==4.4.0 +flashinfer-python==0.6.3 +click==8.3.1 +pytz==2025.2 +wheel==0.46.3 +ipykernel==7.2.0 +stack-data==0.6.3 +absl-py==2.4.0 +wandb==0.25.0 +pandas==2.3.3 +termcolor==3.3.0 +sympy==1.13.1 +joblib==1.5.3 +prompt_toolkit==3.0.52 +attrs==25.4.0 +nvidia-cusolver-cu12==11.6.1.9 +xxhash==3.6.0 +aiohappyeyeballs==2.6.1 +python-dateutil==2.9.0.post0 +frozenlist==1.8.0 +nvidia-ml-py==13.590.48 +multiprocess==0.70.18 +decorator==5.2.1 +cuda-pathfinder==1.3.4 +multidict==6.7.1 +pure_eval==0.2.3 +nvidia-cufft-cu12==11.2.1.3 +omegaconf==2.3.0 +PyYAML==6.0.3 +debugpy==1.8.20 +psutil==7.2.2 +ninja==1.13.0 +nvidia-cublas-cu12==12.4.5.8 +triton==3.2.0 +antlr4-python3-runtime==4.9.3 +peft==0.18.1 +aiosignal==1.4.0 +accelerate==1.12.0 +liger_kernel==0.7.0 +anyio==4.12.1 +decord==0.6.0 +h11==0.16.0 +urllib3==2.6.3 +wcwidth==0.6.0 +networkx==3.4.2 +huggingface_hub==0.36.2 +tzdata==2025.3 +xformers==0.0.29.post3 +tabulate==0.9.0 +certifi==2026.1.4 +tensorboard==2.20.0 +matplotlib-inline==0.2.1 +pip==26.0.1 +smmap==5.0.2 +nvidia-cuda-cupti-cu12==12.4.127 +Markdown==3.10.2 +nvidia-nccl-cu12==2.21.5 +aiohttp==3.13.3 +scikit-learn==1.7.2 +nvidia-cutlass-dsl==4.4.0 +setuptools==82.0.0 +jupyter_core==5.9.1 +asyncio==4.0.0 +einops==0.8.2 +pydantic==2.12.5 +av==16.1.0 +protobuf==6.33.5 +evaluate==0.4.6 +Jinja2==3.1.6 +exceptiongroup==1.3.1 +ipython==8.38.0 +sentry-sdk==2.53.0 +httpx==0.28.1 +msgpack==1.1.2 +platformdirs==4.9.2 +hjson==3.1.0 +torchvision==0.21.0 +parso==0.8.6 +cuda-bindings==13.1.1 +pexpect==4.9.0 +threadpoolctl==3.6.0 +pyarrow==23.0.1 +jedi==0.19.2 +nvidia-cuda-nvrtc-cu12==12.4.127 +Werkzeug==3.1.5 +fsspec==2025.10.0 +pydantic_core==2.41.5 +GitPython==3.1.46 +propcache==0.4.1 +mpmath==1.3.0 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-cusparse-cu12==12.3.1.170 +hf-xet==1.2.0 +charset-normalizer==3.4.4 +dill==0.4.0 +datasets==4.5.0 +py-cpuinfo==9.0.0 +sentencepiece==0.2.1 +latex2sympy2_extended==1.11.0 +jupyter_client==8.8.0 +pyzmq==27.1.0 +numpy==2.2.6 +gitdb==4.0.12 +grpcio==1.78.0 +packaging==26.0 +requests==2.32.5 +nvidia-cudnn-cu12==9.1.0.70 +httpcore==1.0.9 +traitlets==5.14.3 +filelock==3.24.2 +ptyprocess==0.7.0 +pillow==12.1.1 +torch==2.6.0 +more-itertools==10.8.0 +wheel==0.46.3 +jaraco.text==4.0.0 +zipp==3.23.0 +autocommand==2.2.2 +jaraco.context==6.1.0 +jaraco.functools==4.4.0 +tomli==2.4.0 +backports.tarfile==1.2.0 +importlib_metadata==8.7.1 +platformdirs==4.4.0 +packaging==26.0 diff --git a/reward/wandb/run-20260602_172441-w3wdpl8c/files/wandb-metadata.json b/reward/wandb/run-20260602_172441-w3wdpl8c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d0aec2d49a38ed0088e3f08ed12ef391ec7ff852 --- /dev/null +++ b/reward/wandb/run-20260602_172441-w3wdpl8c/files/wandb-metadata.json @@ -0,0 +1,26 @@ +{ + "os": "Linux-5.15.0-144-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2026-06-02T09:24:41.968601Z", + "args": [ + "config=../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml", + "wandb.run_id=w3wdpl8c", + "experiment.function=train", + "evaluation.block_size=None", + "evaluation.top_k=None", + "evaluation.remasking_strategy=None", + "experiment.current_epoch=10" + ], + "program": "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py", + "codePath": "reward/rl_reward_video_reversal.py", + "codePathLocal": "rl_reward_video_reversal.py", + "git": { + "remote": "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git", + "commit": "10b4fd1cba83e4496c20585dc2fd01dc02f20255" + }, + "email": "shixi_runqing@staff.weibo.com", + "root": "/data4/runqing/dLLM-RL/reward", + "host": "b1301b38c75f", + "executable": "/data8/runqing/miniconda/envs/dllm-rl/bin/python", + "writerId": "xvb1lk1cnls4lggnap8yupkndjuv02j1" +} \ No newline at end of file diff --git a/reward/wandb/run-20260602_172441-w3wdpl8c/files/wandb-summary.json b/reward/wandb/run-20260602_172441-w3wdpl8c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..17f457490e8169f9dcfb007c5c460a6f21768257 --- /dev/null +++ b/reward/wandb/run-20260602_172441-w3wdpl8c/files/wandb-summary.json @@ -0,0 +1 @@ +{"reward/avg_response_length":35.435546875,"reward/acc":0.03515625,"train/loss":0.3788445591926574,"train/learning_rate":8.628481651367888e-08,"reward/train_step":3,"_step":10,"reward/unbiased_bonus":0.01875,"_wandb":{"runtime":4615},"reward/t_grpo_bonus":0.3000000000000004,"_timestamp":1.780382672555827e+09,"_runtime":4615} \ No newline at end of file diff --git a/reward/wandb/run-20260602_172441-w3wdpl8c/logs/debug-core.log b/reward/wandb/run-20260602_172441-w3wdpl8c/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..6e07e02f614ccf15c3bd4c47645ec016e4ede1bd --- /dev/null +++ b/reward/wandb/run-20260602_172441-w3wdpl8c/logs/debug-core.log @@ -0,0 +1,18 @@ +{"time":"2026-06-02T17:24:42.051994418+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp9gvuswm2/port-1294603.txt","pid":1294603,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-06-02T17:24:42.052980729+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1294603} +{"time":"2026-06-02T17:24:42.05295701+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1294603-1294761-507784124/socket","Net":"unix"}} +{"time":"2026-06-02T17:24:42.225502385+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-06-02T17:24:42.234398108+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T17:24:42.944982861+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T17:24:43.401648341+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"tvbnpmm4a3rj"} +{"time":"2026-06-02T17:24:43.584902187+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"tvbnpmm4a3rj"} +{"time":"2026-06-02T17:24:43.585467549+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T17:24:43.586306121+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"w3wdpl8c","id":"1(@)"} +{"time":"2026-06-02T17:24:43.586325154+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-06-02T17:24:43.58633788+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-06-02T17:24:43.58634693+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-06-02T17:24:43.586358575+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-06-02T17:24:43.586387557+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-06-02T17:24:43.58640622+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-06-02T17:24:43.586544679+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1294603-1294761-507784124/socket","Net":"unix"}} +{"time":"2026-06-02T17:24:43.586605076+08:00","level":"INFO","msg":"server is closed"} diff --git a/reward/wandb/run-20260602_172441-w3wdpl8c/logs/debug-internal.log b/reward/wandb/run-20260602_172441-w3wdpl8c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c5064e2c0770149c7ce46761e3504bb565e86411 --- /dev/null +++ b/reward/wandb/run-20260602_172441-w3wdpl8c/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2026-06-02T17:24:42.234571344+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-06-02T17:24:42.944524547+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"} +{"time":"2026-06-02T17:24:42.944677241+08:00","level":"INFO","msg":"stream: created new stream","id":"w3wdpl8c"} +{"time":"2026-06-02T17:24:42.944745457+08:00","level":"INFO","msg":"handler: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T17:24:42.944959733+08:00","level":"INFO","msg":"stream: started","id":"w3wdpl8c"} +{"time":"2026-06-02T17:24:42.945007497+08:00","level":"INFO","msg":"writer: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T17:24:42.945007633+08:00","level":"INFO","msg":"sender: started","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T17:24:42.95026804+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-06-02T17:24:43.164047313+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2026-06-02T17:24:43.40037441+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":10,"current":11} +{"time":"2026-06-02T17:24:43.527418509+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f477cbfaf3c796f63c2c6b147923637aa9d39c2ef165397e00f0347b4aedcfbb"} +{"time":"2026-06-02T17:24:43.527558674+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json18B53959535DED314b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_172441-w3wdpl8c/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=f477cbfaf3c796f63c2c6b147923637aa9d39c2ef165397e00f0347b4aedcfbb, Size: 326}"} +{"time":"2026-06-02T17:24:43.531978077+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2d22c426c3a8cb3df871d4138ec60c5e5863a0764dc67fa7be018af066eb507d"} +{"time":"2026-06-02T17:24:43.532064488+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.lognetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log18B5395954F87C9F29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_172441-w3wdpl8c/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=2d22c426c3a8cb3df871d4138ec60c5e5863a0764dc67fa7be018af066eb507d, Size: 110}"} +{"time":"2026-06-02T17:24:43.532234165+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=954de03a03bdd5dbb57e0157732c203c0acc96a16b859757224290347990c459"} +{"time":"2026-06-02T17:24:43.532379214+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0509f173707f2892defb0f21c9e2e6c47bb334b7b458a0787438c6958e4d6dea"} +{"time":"2026-06-02T17:24:43.532364657+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.jsonnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json18B53959531774C617e81bf8-654d-4ce6-b556-0f7349513ee1","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_172441-w3wdpl8c/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=954de03a03bdd5dbb57e0157732c203c0acc96a16b859757224290347990c459, Size: 990}"} +{"time":"2026-06-02T17:24:43.532521566+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yamlnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml18B53959551314DE29678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_172441-w3wdpl8c/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=0509f173707f2892defb0f21c9e2e6c47bb334b7b458a0787438c6958e4d6dea, Size: 5727}"} +{"time":"2026-06-02T17:24:43.533561577+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=de1893dd09a59cfaee0fc988032544ba670e5cfec6f34267ca7c684af523dd17"} +{"time":"2026-06-02T17:24:43.533727454+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txtnetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt18B539595512B80629678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_172441-w3wdpl8c/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=de1893dd09a59cfaee0fc988032544ba670e5cfec6f34267ca7c684af523dd17, Size: 2982}"} +{"time":"2026-06-02T17:24:43.537320756+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5d66cb946d0534a86d8c42f5746a6dcc029fe6851157d4b20ce1469e3e0430db"} +{"time":"2026-06-02T17:24:43.537446239+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: \nSignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.pynetdisk/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py18B5395954E3503729678716-7d96-4915-b26c-b7113941467d","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260602_172441-w3wdpl8c/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260602%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260602T092443Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=5d66cb946d0534a86d8c42f5746a6dcc029fe6851157d4b20ce1469e3e0430db, Size: 14790}"} +{"time":"2026-06-02T17:24:43.53774191+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-06-02T17:24:43.580386285+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-06-02T17:24:43.585495443+08:00","level":"INFO","msg":"stream: closing","id":"w3wdpl8c"} +{"time":"2026-06-02T17:24:43.585516293+08:00","level":"INFO","msg":"handler: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T17:24:43.585603233+08:00","level":"INFO","msg":"sender: closed","stream_id":"w3wdpl8c"} +{"time":"2026-06-02T17:24:43.585618153+08:00","level":"INFO","msg":"stream: closed","id":"w3wdpl8c"} diff --git a/reward/wandb/run-20260602_172441-w3wdpl8c/logs/debug.log b/reward/wandb/run-20260602_172441-w3wdpl8c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6e16ecb92b668305075c15a5df00b82d2ab3258c --- /dev/null +++ b/reward/wandb/run-20260602_172441-w3wdpl8c/logs/debug.log @@ -0,0 +1,25 @@ +2026-06-02 17:24:41,971 INFO MainThread:1294603 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-06-02 17:24:41,972 INFO MainThread:1294603 [wandb_setup.py:_flush():81] Configure stats pid to 1294603 +2026-06-02 17:24:41,972 INFO MainThread:1294603 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-06-02 17:24:41,972 INFO MainThread:1294603 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_172441-w3wdpl8c/logs/debug.log +2026-06-02 17:24:41,972 INFO MainThread:1294603 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260602_172441-w3wdpl8c/logs/debug-internal.log +2026-06-02 17:24:41,972 INFO MainThread:1294603 [wandb_init.py:init():844] calling init triggers +2026-06-02 17:24:41,972 INFO MainThread:1294603 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'w3wdpl8c', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_revvideo_orig10k_4gpu_len128', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 10, 'experiment.save_every': 5, 'experiment.eval_every': 5, 'experiment.current_epoch': 10, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/hf/lavida-llada-v1.0-instruct', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_sft_train_cleaned_10k', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.05, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 5e-07, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 5e-07, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_revvideo_orig10k_4gpu_len128.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}} +2026-06-02 17:24:41,972 INFO MainThread:1294603 [wandb_init.py:init():892] starting backend +2026-06-02 17:24:42,225 INFO MainThread:1294603 [wandb_init.py:init():895] sending inform_init request +2026-06-02 17:24:42,232 INFO MainThread:1294603 [wandb_init.py:init():903] backend started and connected +2026-06-02 17:24:42,234 INFO MainThread:1294603 [wandb_init.py:init():973] updated telemetry +2026-06-02 17:24:42,239 INFO MainThread:1294603 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-06-02 17:24:43,158 INFO MainThread:1294603 [wandb_init.py:init():1037] run resumed +2026-06-02 17:24:43,162 INFO MainThread:1294603 [wandb_init.py:init():1042] starting run threads in backend +2026-06-02 17:24:43,258 INFO MainThread:1294603 [wandb_run.py:_console_start():2524] atexit reg +2026-06-02 17:24:43,258 INFO MainThread:1294603 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-06-02 17:24:43,258 INFO MainThread:1294603 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-06-02 17:24:43,258 INFO MainThread:1294603 [wandb_run.py:_redirect():2465] Redirects installed. +2026-06-02 17:24:43,262 INFO MainThread:1294603 [wandb_init.py:init():1082] run started, returning control to user process +2026-06-02 17:24:43,400 INFO MainThread:1294603 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_revvideo_orig10k_4gpu_len128/w3wdpl8c +2026-06-02 17:24:43,400 INFO MainThread:1294603 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0 +2026-06-02 17:24:43,401 INFO MainThread:1294603 [wandb_run.py:_restore():2472] restore +2026-06-02 17:24:43,401 INFO MainThread:1294603 [wandb_run.py:_restore():2478] restore done +2026-06-02 17:24:43,584 INFO MainThread:1294603 [wandb_run.py:_footer_sync_info():3868] logging synced files diff --git a/reward/wandb/run-20260602_172441-w3wdpl8c/run-w3wdpl8c.wandb b/reward/wandb/run-20260602_172441-w3wdpl8c/run-w3wdpl8c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4cdefd660bc441a78c5454bf811a3a3005e386b9 Binary files /dev/null and b/reward/wandb/run-20260602_172441-w3wdpl8c/run-w3wdpl8c.wandb differ