Add files using upload-large-folder tool

reward/wandb/run-20260529_194707-o3l1vpla/files/code/reward/rl_reward_video_reversal.py

@@ -0,0 +1,339 @@
+import json
+import math_utils_v
+import nest_asyncio
+from concurrent.futures import ThreadPoolExecutor
+import asyncio
+import re
+from termcolor import cprint
+from omegaconf import MISSING
+from omegaconf import DictConfig, ListConfig, OmegaConf
+from typing import List, Union
+import os
+from collections import defaultdict
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, REPO_ROOT)
+
+import wandb
+from train.utils import flatten_omega_conf, ensure_wandb_run_id
+
+def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
+    p = sample.get(f"{media_type}_abs_path")
+    if p is None:
+        p = sample.get(media_type) or sample.get(f"{media_type}_path")
+        if p is None:
+            return None
+        if not os.path.isabs(p) and dataset_root:
+            p = os.path.join(dataset_root, p)
+    try:
+        return os.path.abspath(p)
+    except Exception:
+        return p
+
+def get_config():
+    cli_conf = OmegaConf.from_cli()
+    yaml_conf = OmegaConf.load(cli_conf.config)
+    conf = OmegaConf.merge(yaml_conf, cli_conf)
+    return conf
+
+def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]:
+    token = '<|endoftext|>'
+    if isinstance(obj, str):
+        return obj.replace(token, '')
+    if isinstance(obj, list):
+        return [s.replace(token, '') if isinstance(s, str) else s for s in obj]
+    raise TypeError("Expected str or list of str")
+
+
+if __name__ == "__main__":
+
+    config = get_config()
+    run = None
+    if config.experiment.function == "train":
+        existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None)
+        run_id = ensure_wandb_run_id(config)
+        resume_wandb_run = config.wandb.resume
+        if existing_run_id is None:
+            resume_wandb_run = False
+        wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
+        run = wandb.init(
+            project=config.experiment.project,
+            entity=config.wandb.get("entity", None),
+            id=run_id,
+            resume=resume_wandb_run,
+            config=wandb_config,
+            reinit=True,
+            name=config.experiment.project,
+        )
+
+    project_name = config.experiment.project
+    
+    if config.experiment.current_epoch == 1:
+        pretrained_model = config.model.pretrained_model
+    else:
+        pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name
+    
+    if config.experiment.function == "train":
+        shrink = config.training.shrink
+        dataset = config.dataset.train_dataset
+        outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}"
+        
+    elif config.experiment.function == "evaluation":
+        dataset = config.evaluation.eval_dataset
+        outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}"
+    
+    file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json"
+
+    with open(file_name, 'r') as f:
+        data = json.load(f)
+
+    index_list = []
+    extracted_output_list = []
+    response_length_list = []
+    for i in range(len(data)):
+        data[i]["correctness"] = []
+        response_length_list += data[i]["response_length"]
+        index_list += [i] * len(data[i]["extracted_output"])
+        extracted_output_list += remove_endoftext(data[i]["extracted_output"])
+
+    nest_asyncio.apply()
+    
+    async def get_correctness_all():
+        executor = ThreadPoolExecutor(max_workers=64)
+        tasks = []
+        
+        for p in range(len(index_list)):
+            pred = extracted_output_list[p]
+            idx = index_list[p]
+            
+            # 选择题正确性匹配逻辑
+            if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse":
+                gt = data[idx].get("response", "")
+                
+                async def mcq_check(p_str, g_str):
+                    if p_str in g_str or g_str in p_str:
+                        return True
+
+                    p_str = str(p_str).strip()
+                    g_str = str(g_str).strip()
+                    p_match = re.search(r'^([A-Z])[\.\s]', p_str)
+                    g_match = re.search(r'^([A-Z])[\.\s]', g_str)
+                    if p_match and g_match:
+                        return p_match.group(1) == g_match.group(1)
+                    return g_str in p_str
+                
+                tasks.append(mcq_check(pred, gt))
+            else:
+                gt = data[idx].get("ground_truth_answer", "")
+                tasks.append(math_utils_v.is_equal(pred, gt, executor))
+                
+        return await asyncio.gather(*tasks)
+
+    correctness_all = asyncio.run(get_correctness_all())
+    for p, ok in enumerate(correctness_all):
+        index_i = index_list[p]
+        data[index_i]["correctness"].append(bool(ok))
+
+    # =========================================================================
+    # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励
+    # =========================================================================
+    
+    # 1. 按照 data_id 分组映射索引
+    grouped_data = defaultdict(lambda: {
+        "forward_False": [], "backward_False": [], 
+        "forward_True": [],  "backward_True": []
+    })
+
+    for i in range(len(data)):
+        d_id = data[i].get("data_id")
+        if d_id is None:
+            continue
+        r_type = data[i].get("reversal_type", "forward")
+        is_shuf = str(data[i].get("is_shuffled", False))
+        grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i)
+
+    # 计算并应用群组奖励
+    all_temporal_bonuses = []  # 新增：记录所有的时序红利
+    all_unbiased_bonuses = []  # 新增：记录所有的无偏红利
+
+    # 2. 计算并应用群组奖励
+    for d_id, group in grouped_data.items():
+        
+        def get_acc(key):
+            if not group[key]: return 0.0
+            idx = group[key][0]
+            corr = data[idx]["correctness"]
+            return sum(corr) / len(corr) if len(corr) > 0 else 0.0
+
+        # 获取四组情况的正确率
+        acc_f_ord = get_acc("forward_False")
+        acc_f_shuf = get_acc("forward_True")
+        acc_b_ord = get_acc("backward_False")
+        acc_b_shuf = get_acc("backward_True")
+
+        # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3
+        f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0
+        b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0
+
+        # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3
+        unbiased_bonus = 0.0
+        if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1):
+            unbiased_bonus = 0.3
+
+        # 新增：将当前组的奖励记录下来用于统计
+        if group["forward_False"]:
+            all_temporal_bonuses.append(f_temporal_bonus)
+            all_unbiased_bonuses.append(unbiased_bonus)
+        if group["backward_False"]:
+            all_temporal_bonuses.append(b_temporal_bonus)
+            all_unbiased_bonuses.append(unbiased_bonus)
+
+        # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本
+        def apply_bonus(key, temporal_bonus, unbiased_bonus):
+            if not group[key]: return
+            idx = group[key][0]
+            raw_rewards = []
+            for is_correct in data[idx]["correctness"]:
+                if is_correct:
+                    # 基础分 1.0 + 时序红利 + 无偏红利
+                    raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus)
+                else:
+                    raw_rewards.append(0.0)
+            data[idx]["raw_rewards"] = raw_rewards
+
+        apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus)
+        apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus)
+
+    # =========================================================================
+
+    def z_score_normalize(lst):
+        mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0
+        std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0
+        if std == 0:
+            return [0 for _ in lst]
+        return [(x - mean) / std for x in lst]
+
+    final_data = []
+    ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据
+    
+    for i in range(len(data)):
+        # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习
+        if data[i].get("is_shuffled", False):
+            continue
+            
+        ordered_correctness.extend(data[i]["correctness"])
+
+        # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数
+        raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]])
+        lengths = data[i]["response_length"]
+        
+        # 长度惩罚
+        if config.reward.strict_len_check:
+            for j in range(len(lengths)):
+                if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5:
+                    raw_rewards[j] = 0.0
+                if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5:
+                    raw_rewards[j] = 0.0
+
+        # Z-Score 优势归一化
+        rewards = z_score_normalize(raw_rewards)
+        data[i]["rewards"] = rewards
+
+        if config.experiment.function == "train":
+            proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0
+            # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本
+            if proportion > 0.95 or proportion < 0.05:
+                continue
+
+            for j in range(len(rewards)):
+                data_i = {}
+                data_i["prompt"] = data[i]["prompt"]
+                data_i["reward"] = rewards[j]
+                data_i["response"] = data[i]["full_output"][j]
+                data_i["step_map"] = data[i]["step_map"][j]
+                
+                # 保留其他需要的上下文 ID
+                data_i["data_id"] = data[i].get("data_id")
+                data_i["reversal_type"] = data[i].get("reversal_type")
+                
+                if "image_token_ids" in data[i]:
+                    data_i["image_token_ids"] = data[i]["image_token_ids"]
+                
+                p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video")
+                if p_video_abs is not None:
+                    data_i["video_abs_path"] = p_video_abs
+                
+                p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image")
+                if p_image_abs is not None:
+                    data_i["image_abs_path"] = p_image_abs
+
+                final_data.append(data_i)
+
+        if config.experiment.function == "evaluation":
+            data[i]["step_map"] = []
+
+    if config.experiment.function == "train":
+        with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f:
+            json.dump(final_data, f, indent=2, ensure_ascii=False)
+
+
+    os.makedirs(os.path.dirname(file_name), exist_ok=True)
+    with open(file_name, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+
+
+    outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt"
+    os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True)
+    with open(outputs_result_name, "a") as f:
+        # Save + print
+        def save_and_print(text):
+            cprint("\n\n\n" + text, color="green")
+            f.write(text + "\n")
+        
+        # 只计算有序组（有效学习组）的准确率来作为监控指标
+        acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan")
+        avg_len = sum(response_length_list)/len(response_length_list)
+
+        # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值
+        avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0
+        avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0
+
+        output_text = f"train step: {config.experiment.current_epoch}  "
+        if config.experiment.function == "train":
+            if config.model.model_base != "sdar" and config.model.model_base != "trado":
+                output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy}  block_size: {config.rollout.block_size}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+            else:
+                output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy}  top_k: {config.rollout.top_k}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+        else:
+            if config.model.model_base != "sdar" and config.model.model_base != "trado":
+                output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy}  block_size: {config.evaluation.block_size}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+            else:
+                output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy}  top_k: {config.evaluation.top_k}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+        save_and_print(output_text)
+
+    if config.experiment.function == "train":
+        csv_file_path = f"../{project_name}/results/all_rewards.csv"
+        
+        # 如果是第1个 epoch/step，或者是新文件，就先写入表头
+        write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1
+        
+        with open(csv_file_path, "a", encoding="utf-8") as f_csv:
+            if write_header:
+                f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n")
+            
+            # 按行追加当前 step 的数据
+            f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n")
+
+        if run is not None:
+            wandb.log(
+                {
+                    "reward/acc": acc,
+                    "reward/t_grpo_bonus": avg_temp,
+                    "reward/unbiased_bonus": avg_unbias,
+                    "reward/avg_response_length": avg_len,
+                    "reward/train_step": config.experiment.current_epoch,
+                },
+                step=config.experiment.current_epoch,
+            )
+            wandb.finish()

reward/wandb/run-20260529_194707-o3l1vpla/files/config.yaml

@@ -0,0 +1,273 @@
+_wandb:
+    value:
+        cli_version: 0.25.0
+        code_path: code/reward/rl_reward_video_reversal.py
+        e:
+            9pqfh5lrgyl0srsilyb4o7f1khpc7ymu:
+                args:
+                    - config=../configs/rl_lavida_video_reversal.yaml
+                    - wandb.run_id=o3l1vpla
+                    - experiment.function=train
+                    - evaluation.block_size=None
+                    - evaluation.top_k=None
+                    - evaluation.remasking_strategy=None
+                    - experiment.current_epoch=1
+                codePath: reward/rl_reward_video_reversal.py
+                codePathLocal: rl_reward_video_reversal.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "806578724864"
+                        used: "623847804928"
+                email: shixi_runqing@staff.weibo.com
+                executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python
+                git:
+                    commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255
+                    remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git
+                gpu: NVIDIA H800
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-2775489f-18c8-2328-8829-c46238575379
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd
+                host: b1301b38c75f
+                memory:
+                    total: "2164113764352"
+                os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35
+                program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py
+                python: CPython 3.10.19
+                root: /data4/runqing/dLLM-RL/reward
+                startedAt: "2026-05-29T11:47:07.400494Z"
+                writerId: 9pqfh5lrgyl0srsilyb4o7f1khpc7ymu
+        m: []
+        python_version: 3.10.19
+        t:
+            "1":
+                - 1
+                - 41
+            "2":
+                - 1
+                - 41
+            "3":
+                - 2
+                - 13
+                - 14
+                - 16
+                - 61
+            "4": 3.10.19
+            "5": 0.25.0
+            "10":
+                - 20
+            "12": 0.25.0
+            "13": linux-x86_64
+config:
+    value: ../configs/rl_lavida_video_reversal.yaml
+dataset.data_type:
+    value: video_reverse
+dataset.optimization_data:
+    value: rl_data
+dataset.train_dataset:
+    value: video_reversal_rl_train
+evaluation.batch_size:
+    value: 1
+evaluation.block_size:
+    value: None
+evaluation.data_type:
+    value: video_reverse
+evaluation.dynamic_threshold:
+    value: 0.9
+evaluation.eval_dataset:
+    value: video_reversal_rl_eval
+evaluation.further_horizon:
+    value: 128
+evaluation.if_eval:
+    value: false
+evaluation.max_gen_length:
+    value: 128
+evaluation.num_response_per_task:
+    value: 3
+evaluation.remasking_strategy:
+    value: None
+evaluation.steps:
+    value: 128
+evaluation.target:
+    value: confidence
+evaluation.temperature:
+    value: 0.1
+evaluation.top_k:
+    value: None
+evaluation.use_cache:
+    value: true
+execute.num_chunk:
+    value: 128
+experiment.current_epoch:
+    value: 1
+experiment.deepspeed_file:
+    value: 1_node_8_gpus_deepspeed_zero2
+experiment.eval_every:
+    value: 5
+experiment.function:
+    value: train
+experiment.node_index:
+    value: 0
+experiment.num_node:
+    value: 1
+experiment.project:
+    value: rl_lavida_video_reversal
+experiment.save_every:
+    value: 10
+experiment.start_from_scratch:
+    value: true
+experiment.total_step:
+    value: 30
+lr_scheduler.params.learning_rate:
+    value: 1e-06
+lr_scheduler.params.min_lr_scale:
+    value: 1
+lr_scheduler.params.warmup_steps:
+    value: 0
+lr_scheduler.scheduler:
+    value: cosine
+model.image_resolution:
+    value: 512
+model.model_base:
+    value: lavida
+model.optimized_name:
+    value: optimized
+model.pretrained_model:
+    value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized
+model.vq_model_path:
+    value: /data4/runqing/hf/magvitv2
+optimizer.name:
+    value: adamw
+optimizer.params.beta1:
+    value: 0.9
+optimizer.params.beta2:
+    value: 0.999
+optimizer.params.epsilon:
+    value: 1e-08
+optimizer.params.learning_rate:
+    value: 1e-06
+optimizer.params.scale_lr:
+    value: false
+optimizer.params.weight_decay:
+    value: 0
+reward.answer_must_in_box:
+    value: true
+reward.strict_len_check:
+    value: false
+rollout.batch_size:
+    value: 1
+rollout.block_size:
+    value: 32
+rollout.dynamic_threshold:
+    value: 0.9
+rollout.further_horizon:
+    value: 128
+rollout.max_gen_length:
+    value: 128
+rollout.num_response_per_task:
+    value: 8
+rollout.num_task_per_step:
+    value: 128
+rollout.remasking_strategy:
+    value: low_confidence_dynamic
+rollout.steps:
+    value: 128
+rollout.target:
+    value: confidence
+rollout.temperature:
+    value: 0.8
+rollout.use_cache:
+    value: true
+system.HF_HOME:
+    value: null
+system.HTTP_PROXY:
+    value: null
+system.base_dir:
+    value: /data4/runqing/dLLM-RL
+system.env_name:
+    value: /data8/runqing/miniconda/envs/dllm-rl
+system.envs_dir:
+    value: /data8/runqing/miniconda/envs/dllm-rl
+training.batch_size_lm:
+    value: 1
+training.beta:
+    value: 0.01
+training.enable_tf32:
+    value: true
+training.eps:
+    value: 0.2
+training.gradient_accumulation_steps:
+    value: 2
+training.lower_p:
+    value: 0.35
+training.mask_times_per_sample:
+    value: 2
+training.max_gen_length:
+    value: 128
+training.max_grad_norm:
+    value: 1
+training.max_prompt_len:
+    value: 512
+training.method:
+    value: random_masking
+training.mixed_precision:
+    value: bf16
+training.num_train_epochs:
+    value: 1
+training.post_num:
+    value: 1
+training.seed:
+    value: 10086
+training.shrink:
+    value: 8
+training.upper_p:
+    value: 0.9
+training.use_kl_estimator_k3:
+    value: true
+wandb.entity:
+    value: null
+wandb.resume:
+    value: auto
+wandb.run_id:
+    value: o3l1vpla

reward/wandb/run-20260529_194707-o3l1vpla/files/output.log

@@ -0,0 +1,4 @@
+
+
+
+train step: 1  remasking_strategy: low_confidence_dynamic  block_size: 32  acc: 0.4824  avg length: 126.07

reward/wandb/run-20260529_194707-o3l1vpla/files/requirements.txt

@@ -0,0 +1,157 @@
+transformers==4.52.4
+nvidia-nvtx-cu12==12.4.127
+deepspeed==0.18.6
+typing-inspection==0.4.2
+scipy==1.15.3
+MarkupSafe==3.0.3
+nvidia-cuda-runtime-cu12==12.4.127
+detoxify==0.5.2
+yarl==1.22.0
+tornado==6.5.4
+nvidia-curand-cu12==10.3.5.147
+nest-asyncio==1.6.0
+safetensors==0.7.0
+async-timeout==5.0.1
+typing_extensions==4.15.0
+asttokens==3.0.1
+cuda-python==13.1.1
+executing==2.2.1
+nvidia-cudnn-frontend==1.18.0
+comm==0.2.3
+tokenizers==0.21.4
+six==1.17.0
+Pygments==2.19.2
+nvidia-cusparselt-cu12==0.6.2
+apache-tvm-ffi==0.1.8.post2
+flash_attn==2.7.4.post1
+tqdm==4.67.3
+tensorboard-data-server==0.7.2
+annotated-types==0.7.0
+math-verify==0.9.0
+idna==3.11
+regex==2026.1.15
+nvidia-cutlass-dsl-libs-base==4.4.0
+flashinfer-python==0.6.3
+click==8.3.1
+pytz==2025.2
+wheel==0.46.3
+ipykernel==7.2.0
+stack-data==0.6.3
+absl-py==2.4.0
+wandb==0.25.0
+pandas==2.3.3
+termcolor==3.3.0
+sympy==1.13.1
+joblib==1.5.3
+prompt_toolkit==3.0.52
+attrs==25.4.0
+nvidia-cusolver-cu12==11.6.1.9
+xxhash==3.6.0
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+frozenlist==1.8.0
+nvidia-ml-py==13.590.48
+multiprocess==0.70.18
+decorator==5.2.1
+cuda-pathfinder==1.3.4
+multidict==6.7.1
+pure_eval==0.2.3
+nvidia-cufft-cu12==11.2.1.3
+omegaconf==2.3.0
+PyYAML==6.0.3
+debugpy==1.8.20
+psutil==7.2.2
+ninja==1.13.0
+nvidia-cublas-cu12==12.4.5.8
+triton==3.2.0
+antlr4-python3-runtime==4.9.3
+peft==0.18.1
+aiosignal==1.4.0
+accelerate==1.12.0
+liger_kernel==0.7.0
+anyio==4.12.1
+decord==0.6.0
+h11==0.16.0
+urllib3==2.6.3
+wcwidth==0.6.0
+networkx==3.4.2
+huggingface_hub==0.36.2
+tzdata==2025.3
+xformers==0.0.29.post3
+tabulate==0.9.0
+certifi==2026.1.4
+tensorboard==2.20.0
+matplotlib-inline==0.2.1
+pip==26.0.1
+smmap==5.0.2
+nvidia-cuda-cupti-cu12==12.4.127
+Markdown==3.10.2
+nvidia-nccl-cu12==2.21.5
+aiohttp==3.13.3
+scikit-learn==1.7.2
+nvidia-cutlass-dsl==4.4.0
+setuptools==82.0.0
+jupyter_core==5.9.1
+asyncio==4.0.0
+einops==0.8.2
+pydantic==2.12.5
+av==16.1.0
+protobuf==6.33.5
+evaluate==0.4.6
+Jinja2==3.1.6
+exceptiongroup==1.3.1
+ipython==8.38.0
+sentry-sdk==2.53.0
+httpx==0.28.1
+msgpack==1.1.2
+platformdirs==4.9.2
+hjson==3.1.0
+torchvision==0.21.0
+parso==0.8.6
+cuda-bindings==13.1.1
+pexpect==4.9.0
+threadpoolctl==3.6.0
+pyarrow==23.0.1
+jedi==0.19.2
+nvidia-cuda-nvrtc-cu12==12.4.127
+Werkzeug==3.1.5
+fsspec==2025.10.0
+pydantic_core==2.41.5
+GitPython==3.1.46
+propcache==0.4.1
+mpmath==1.3.0
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-cusparse-cu12==12.3.1.170
+hf-xet==1.2.0
+charset-normalizer==3.4.4
+dill==0.4.0
+datasets==4.5.0
+py-cpuinfo==9.0.0
+sentencepiece==0.2.1
+latex2sympy2_extended==1.11.0
+jupyter_client==8.8.0
+pyzmq==27.1.0
+numpy==2.2.6
+gitdb==4.0.12
+grpcio==1.78.0
+packaging==26.0
+requests==2.32.5
+nvidia-cudnn-cu12==9.1.0.70
+httpcore==1.0.9
+traitlets==5.14.3
+filelock==3.24.2
+ptyprocess==0.7.0
+pillow==12.1.1
+torch==2.6.0
+more-itertools==10.8.0
+wheel==0.46.3
+jaraco.text==4.0.0
+zipp==3.23.0
+autocommand==2.2.2
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+tomli==2.4.0
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+platformdirs==4.4.0
+packaging==26.0

reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-metadata.json

@@ -0,0 +1,98 @@
+{
+  "os":  "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.19",
+  "startedAt":  "2026-05-29T11:47:07.400494Z",
+  "args":  [
+    "config=../configs/rl_lavida_video_reversal.yaml",
+    "wandb.run_id=o3l1vpla",
+    "experiment.function=train",
+    "evaluation.block_size=None",
+    "evaluation.top_k=None",
+    "evaluation.remasking_strategy=None",
+    "experiment.current_epoch=1"
+  ],
+  "program":  "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py",
+  "codePath":  "reward/rl_reward_video_reversal.py",
+  "codePathLocal":  "rl_reward_video_reversal.py",
+  "git":  {
+    "remote":  "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
+    "commit":  "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
+  },
+  "email":  "shixi_runqing@staff.weibo.com",
+  "root":  "/data4/runqing/dLLM-RL/reward",
+  "host":  "b1301b38c75f",
+  "executable":  "/data8/runqing/miniconda/envs/dllm-rl/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H800",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "806578724864",
+      "used":  "623847804928"
+    }
+  },
+  "memory":  {
+    "total":  "2164113764352"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2775489f-18c8-2328-8829-c46238575379"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
+    }
+  ],
+  "cudaVersion":  "12.8",
+  "writerId":  "9pqfh5lrgyl0srsilyb4o7f1khpc7ymu"
+}

reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-summary.json

@@ -0,0 +1 @@
+{"reward/unbiased_bonus":0.07499999999999998,"_wandb":{"runtime":0},"reward/acc":0.482421875,"reward/t_grpo_bonus":0.2765625000000003,"reward/train_step":1,"_timestamp":1.7800552284795003e+09,"_runtime":0.276835887,"_step":1,"reward/avg_response_length":126.07421875}

reward/wandb/run-20260529_194707-o3l1vpla/logs/debug-core.log

@@ -0,0 +1,18 @@
+{"time":"2026-05-29T19:47:07.473481181+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpbw7abz65/port-1212640.txt","pid":1212640,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-05-29T19:47:07.474437183+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1212640-1212740-3778981817/socket","Net":"unix"}}
+{"time":"2026-05-29T19:47:07.474559698+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1212640}
+{"time":"2026-05-29T19:47:07.655012851+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-05-29T19:47:07.664220576+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"o3l1vpla","id":"1(@)"}
+{"time":"2026-05-29T19:47:07.947077344+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"o3l1vpla","id":"1(@)"}
+{"time":"2026-05-29T19:47:08.483101048+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"lu9yljxzchbl"}
+{"time":"2026-05-29T19:47:08.683766011+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"lu9yljxzchbl"}
+{"time":"2026-05-29T19:47:08.685066243+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"o3l1vpla","id":"1(@)"}
+{"time":"2026-05-29T19:47:08.685504424+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"o3l1vpla","id":"1(@)"}
+{"time":"2026-05-29T19:47:08.685645737+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-05-29T19:47:08.685676362+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-05-29T19:47:08.685684169+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-05-29T19:47:08.685710476+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-05-29T19:47:08.685743906+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1212640-1212740-3778981817/socket","Net":"unix"}}
+{"time":"2026-05-29T19:47:08.685770189+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-05-29T19:47:08.68577738+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-05-29T19:47:08.685784402+08:00","level":"INFO","msg":"server is closed"}

reward/wandb/run-20260529_194707-o3l1vpla/logs/debug-internal.log

@@ -0,0 +1,27 @@
+{"time":"2026-05-29T19:47:07.664329293+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
+{"time":"2026-05-29T19:47:07.946806555+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
+{"time":"2026-05-29T19:47:07.946916582+08:00","level":"INFO","msg":"stream: created new stream","id":"o3l1vpla"}
+{"time":"2026-05-29T19:47:07.946954411+08:00","level":"INFO","msg":"handler: started","stream_id":"o3l1vpla"}
+{"time":"2026-05-29T19:47:07.94706896+08:00","level":"INFO","msg":"stream: started","id":"o3l1vpla"}
+{"time":"2026-05-29T19:47:07.947088794+08:00","level":"INFO","msg":"writer: started","stream_id":"o3l1vpla"}
+{"time":"2026-05-29T19:47:07.947090844+08:00","level":"INFO","msg":"sender: started","stream_id":"o3l1vpla"}
+{"time":"2026-05-29T19:47:07.948297319+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
+{"time":"2026-05-29T19:47:08.206739239+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-29T19:47:08.62874404+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a47ee7cc56a7d98ad3f50a5a4c0649595b715b64e89e47637ac5aff422ec7383"}
+{"time":"2026-05-29T19:47:08.628809977+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=883c4209c5d742770b3a6d59ac71abe2715046eb59ec69dccaa1a5b567ba579b"}
+{"time":"2026-05-29T19:47:08.628827792+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-metadata.json</Resource><RequestId>18B406CC9D558263</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a47ee7cc56a7d98ad3f50a5a4c0649595b715b64e89e47637ac5aff422ec7383, Size: 2849}"}
+{"time":"2026-05-29T19:47:08.628868986+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/o3l1vpla/config.yaml</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/config.yaml</Resource><RequestId>18B406CC9F0C3E1B</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=883c4209c5d742770b3a6d59ac71abe2715046eb59ec69dccaa1a5b567ba579b, Size: 7755}"}
+{"time":"2026-05-29T19:47:08.630071945+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cde00db00b44971fb11a4d87808dfbc4eeeac5cbe9873b1fa83739955ab417da"}
+{"time":"2026-05-29T19:47:08.630148543+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/o3l1vpla/output.log</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/output.log</Resource><RequestId>18B406CC9F05F219</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=cde00db00b44971fb11a4d87808dfbc4eeeac5cbe9873b1fa83739955ab417da, Size: 110}"}
+{"time":"2026-05-29T19:47:08.63086663+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e4bb8822ef29df26949ff5ddef85264847f7f4b05c641a324b91d1585449ba55"}
+{"time":"2026-05-29T19:47:08.630961997+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/o3l1vpla/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/requirements.txt</Resource><RequestId>18B406CC9DD7FC86</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e4bb8822ef29df26949ff5ddef85264847f7f4b05c641a324b91d1585449ba55, Size: 2982}"}
+{"time":"2026-05-29T19:47:08.633145978+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7226b6bd23e4ef9db8e48e70a2774674ebb0cbec0bb484247d588b1c4633fbc5"}
+{"time":"2026-05-29T19:47:08.633266177+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-summary.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-summary.json</Resource><RequestId>18B406CC9F223DB3</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7226b6bd23e4ef9db8e48e70a2774674ebb0cbec0bb484247d588b1c4633fbc5, Size: 267}"}
+{"time":"2026-05-29T19:47:08.633747103+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=03e385aef9c6b56540adbe0ddb3f51b263ba0e7776fdc7debbb663e00aa220b0"}
+{"time":"2026-05-29T19:47:08.633803977+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/o3l1vpla/code/reward/rl_reward_video_reversal.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/code/reward/rl_reward_video_reversal.py</Resource><RequestId>18B406CC9F05F1CD</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/o3l1vpla/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T114708Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=03e385aef9c6b56540adbe0ddb3f51b263ba0e7776fdc7debbb663e00aa220b0, Size: 14790}"}
+{"time":"2026-05-29T19:47:08.63416604+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-29T19:47:08.676087931+08:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-05-29T19:47:08.685079701+08:00","level":"INFO","msg":"stream: closing","id":"o3l1vpla"}
+{"time":"2026-05-29T19:47:08.685100599+08:00","level":"INFO","msg":"handler: closed","stream_id":"o3l1vpla"}
+{"time":"2026-05-29T19:47:08.685170573+08:00","level":"INFO","msg":"sender: closed","stream_id":"o3l1vpla"}
+{"time":"2026-05-29T19:47:08.685177465+08:00","level":"INFO","msg":"stream: closed","id":"o3l1vpla"}

reward/wandb/run-20260529_194707-o3l1vpla/logs/debug.log

@@ -0,0 +1,24 @@
+2026-05-29 19:47:07,401 INFO    MainThread:1212640 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
+2026-05-29 19:47:07,402 INFO    MainThread:1212640 [wandb_setup.py:_flush():81] Configure stats pid to 1212640
+2026-05-29 19:47:07,402 INFO    MainThread:1212640 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-29 19:47:07,402 INFO    MainThread:1212640 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug.log
+2026-05-29 19:47:07,402 INFO    MainThread:1212640 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260529_194707-o3l1vpla/logs/debug-internal.log
+2026-05-29 19:47:07,402 INFO    MainThread:1212640 [wandb_init.py:init():844] calling init triggers
+2026-05-29 19:47:07,402 INFO    MainThread:1212640 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'o3l1vpla', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 1, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}}
+2026-05-29 19:47:07,402 INFO    MainThread:1212640 [wandb_init.py:init():892] starting backend
+2026-05-29 19:47:07,656 INFO    MainThread:1212640 [wandb_init.py:init():895] sending inform_init request
+2026-05-29 19:47:07,660 INFO    MainThread:1212640 [wandb_init.py:init():903] backend started and connected
+2026-05-29 19:47:07,662 INFO    MainThread:1212640 [wandb_init.py:init():973] updated telemetry
+2026-05-29 19:47:07,673 INFO    MainThread:1212640 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-29 19:47:08,204 INFO    MainThread:1212640 [wandb_init.py:init():1042] starting run threads in backend
+2026-05-29 19:47:08,315 INFO    MainThread:1212640 [wandb_run.py:_console_start():2524] atexit reg
+2026-05-29 19:47:08,316 INFO    MainThread:1212640 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-05-29 19:47:08,316 INFO    MainThread:1212640 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-05-29 19:47:08,316 INFO    MainThread:1212640 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-05-29 19:47:08,318 INFO    MainThread:1212640 [wandb_init.py:init():1082] run started, returning control to user process
+2026-05-29 19:47:08,479 INFO    MainThread:1212640 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/o3l1vpla
+2026-05-29 19:47:08,480 INFO    MainThread:1212640 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
+2026-05-29 19:47:08,480 INFO    MainThread:1212640 [wandb_run.py:_restore():2472] restore
+2026-05-29 19:47:08,480 INFO    MainThread:1212640 [wandb_run.py:_restore():2478] restore done
+2026-05-29 19:47:08,684 INFO    MainThread:1212640 [wandb_run.py:_footer_sync_info():3868] logging synced files

reward/wandb/run-20260529_221548-fmmkx1a5/files/wandb-metadata.json

@@ -0,0 +1,98 @@
+{
+  "os":  "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.19",
+  "startedAt":  "2026-05-29T14:15:48.972009Z",
+  "args":  [
+    "config=../configs/rl_lavida_video_reversal.yaml",
+    "wandb.run_id=fmmkx1a5",
+    "experiment.function=train",
+    "evaluation.block_size=None",
+    "evaluation.top_k=None",
+    "evaluation.remasking_strategy=None",
+    "experiment.current_epoch=2"
+  ],
+  "program":  "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py",
+  "codePath":  "reward/rl_reward_video_reversal.py",
+  "codePathLocal":  "rl_reward_video_reversal.py",
+  "git":  {
+    "remote":  "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
+    "commit":  "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
+  },
+  "email":  "shixi_runqing@staff.weibo.com",
+  "root":  "/data4/runqing/dLLM-RL/reward",
+  "host":  "b1301b38c75f",
+  "executable":  "/data8/runqing/miniconda/envs/dllm-rl/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H800",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "806578724864",
+      "used":  "624733896704"
+    }
+  },
+  "memory":  {
+    "total":  "2164113764352"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2775489f-18c8-2328-8829-c46238575379"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
+    }
+  ],
+  "cudaVersion":  "12.8",
+  "writerId":  "of7uuhm8djcrujnrkdjlueoxbofh7yt3"
+}

reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug-core.log

@@ -0,0 +1,18 @@
+{"time":"2026-05-29T22:15:49.038496026+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpextmq8yv/port-1544888.txt","pid":1544888,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-05-29T22:15:49.039517557+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1544888}
+{"time":"2026-05-29T22:15:49.039525747+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1544888-1545045-3458517359/socket","Net":"unix"}}
+{"time":"2026-05-29T22:15:49.220936217+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-05-29T22:15:49.231304816+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-29T22:15:49.532027363+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-29T22:15:49.887995417+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"zs4zg8uz72tg"}
+{"time":"2026-05-29T22:15:50.066303944+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"zs4zg8uz72tg"}
+{"time":"2026-05-29T22:15:50.067591053+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-29T22:15:50.068479156+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-29T22:15:50.068513746+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-05-29T22:15:50.068538601+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-05-29T22:15:50.068549089+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-05-29T22:15:50.068566656+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-05-29T22:15:50.068599467+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-05-29T22:15:50.068611716+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-05-29T22:15:50.06874089+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1544888-1545045-3458517359/socket","Net":"unix"}}
+{"time":"2026-05-29T22:15:50.068796137+08:00","level":"INFO","msg":"server is closed"}

reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug-internal.log

@@ -0,0 +1,28 @@
+{"time":"2026-05-29T22:15:49.231529459+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
+{"time":"2026-05-29T22:15:49.531421968+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
+{"time":"2026-05-29T22:15:49.531682346+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"}
+{"time":"2026-05-29T22:15:49.53183915+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-29T22:15:49.532000568+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"}
+{"time":"2026-05-29T22:15:49.532047358+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-29T22:15:49.532051598+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-29T22:15:49.534503641+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
+{"time":"2026-05-29T22:15:49.676991563+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-29T22:15:49.886864995+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":2,"current":23}
+{"time":"2026-05-29T22:15:50.012976753+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1c618949f54f91ead5f029d477b6f3a66b28049d8038323dc4e7c19987324b9d"}
+{"time":"2026-05-29T22:15:50.013148766+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json</Resource><RequestId>18B40EE9CACCD1E7</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=1c618949f54f91ead5f029d477b6f3a66b28049d8038323dc4e7c19987324b9d, Size: 339}"}
+{"time":"2026-05-29T22:15:50.013183236+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c615766f40662e6943a47d0789f9209b0dce3b6b820193907bbc7ec403b423a2"}
+{"time":"2026-05-29T22:15:50.013319357+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml</Resource><RequestId>18B40EE9CAC0BA72</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=c615766f40662e6943a47d0789f9209b0dce3b6b820193907bbc7ec403b423a2, Size: 7994}"}
+{"time":"2026-05-29T22:15:50.013842455+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9e76d2be777d1c7ca83aad7eb2ea22d1cbf6d583c423df7e1d096b094110beb1"}
+{"time":"2026-05-29T22:15:50.013942597+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json</Resource><RequestId>18B40EE9CACDD070</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9e76d2be777d1c7ca83aad7eb2ea22d1cbf6d583c423df7e1d096b094110beb1, Size: 2849}"}
+{"time":"2026-05-29T22:15:50.014336234+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e6a49b3ce09eab10673811108dd02727a560b4aa9b50821a1ce992f363b5e7f7"}
+{"time":"2026-05-29T22:15:50.014497174+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt</Resource><RequestId>18B40EE9CB5CF821</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e6a49b3ce09eab10673811108dd02727a560b4aa9b50821a1ce992f363b5e7f7, Size: 2982}"}
+{"time":"2026-05-29T22:15:50.015699606+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=eefbe5c2c8d9371e1c63c8ec320a10cf528f816dec9c2f86a97a3194a7363fe0"}
+{"time":"2026-05-29T22:15:50.015867408+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log</Resource><RequestId>18B40EE9CB77AB38</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=eefbe5c2c8d9371e1c63c8ec320a10cf528f816dec9c2f86a97a3194a7363fe0, Size: 110}"}
+{"time":"2026-05-29T22:15:50.022155163+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9d1064af61d48bce0c860f3c3c26be81a940a20ec21aab3db9f87403d7af47e4"}
+{"time":"2026-05-29T22:15:50.022205902+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py</Resource><RequestId>18B40EE9CB5CA459</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T141549Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9d1064af61d48bce0c860f3c3c26be81a940a20ec21aab3db9f87403d7af47e4, Size: 14790}"}
+{"time":"2026-05-29T22:15:50.022643716+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-29T22:15:50.062268078+08:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-05-29T22:15:50.067618841+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"}
+{"time":"2026-05-29T22:15:50.067638466+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"}
+{"time":"2026-05-29T22:15:50.067710417+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"}
+{"time":"2026-05-29T22:15:50.06774884+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"}

reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug.log

@@ -0,0 +1,25 @@
+2026-05-29 22:15:48,974 INFO    MainThread:1544888 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
+2026-05-29 22:15:48,974 INFO    MainThread:1544888 [wandb_setup.py:_flush():81] Configure stats pid to 1544888
+2026-05-29 22:15:48,974 INFO    MainThread:1544888 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-29 22:15:48,975 INFO    MainThread:1544888 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug.log
+2026-05-29 22:15:48,975 INFO    MainThread:1544888 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260529_221548-fmmkx1a5/logs/debug-internal.log
+2026-05-29 22:15:48,975 INFO    MainThread:1544888 [wandb_init.py:init():844] calling init triggers
+2026-05-29 22:15:48,975 INFO    MainThread:1544888 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 2, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}}
+2026-05-29 22:15:48,975 INFO    MainThread:1544888 [wandb_init.py:init():892] starting backend
+2026-05-29 22:15:49,221 INFO    MainThread:1544888 [wandb_init.py:init():895] sending inform_init request
+2026-05-29 22:15:49,227 INFO    MainThread:1544888 [wandb_init.py:init():903] backend started and connected
+2026-05-29 22:15:49,228 INFO    MainThread:1544888 [wandb_init.py:init():973] updated telemetry
+2026-05-29 22:15:49,234 INFO    MainThread:1544888 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-29 22:15:49,670 INFO    MainThread:1544888 [wandb_init.py:init():1037] run resumed
+2026-05-29 22:15:49,674 INFO    MainThread:1544888 [wandb_init.py:init():1042] starting run threads in backend
+2026-05-29 22:15:49,764 INFO    MainThread:1544888 [wandb_run.py:_console_start():2524] atexit reg
+2026-05-29 22:15:49,765 INFO    MainThread:1544888 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-05-29 22:15:49,765 INFO    MainThread:1544888 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-05-29 22:15:49,765 INFO    MainThread:1544888 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-05-29 22:15:49,769 INFO    MainThread:1544888 [wandb_init.py:init():1082] run started, returning control to user process
+2026-05-29 22:15:49,886 INFO    MainThread:1544888 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5
+2026-05-29 22:15:49,887 INFO    MainThread:1544888 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
+2026-05-29 22:15:49,887 INFO    MainThread:1544888 [wandb_run.py:_restore():2472] restore
+2026-05-29 22:15:49,887 INFO    MainThread:1544888 [wandb_run.py:_restore():2478] restore done
+2026-05-29 22:15:50,066 INFO    MainThread:1544888 [wandb_run.py:_footer_sync_info():3868] logging synced files

reward/wandb/run-20260530_001305-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py

@@ -0,0 +1,339 @@
+import json
+import math_utils_v
+import nest_asyncio
+from concurrent.futures import ThreadPoolExecutor
+import asyncio
+import re
+from termcolor import cprint
+from omegaconf import MISSING
+from omegaconf import DictConfig, ListConfig, OmegaConf
+from typing import List, Union
+import os
+from collections import defaultdict
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, REPO_ROOT)
+
+import wandb
+from train.utils import flatten_omega_conf, ensure_wandb_run_id
+
+def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
+    p = sample.get(f"{media_type}_abs_path")
+    if p is None:
+        p = sample.get(media_type) or sample.get(f"{media_type}_path")
+        if p is None:
+            return None
+        if not os.path.isabs(p) and dataset_root:
+            p = os.path.join(dataset_root, p)
+    try:
+        return os.path.abspath(p)
+    except Exception:
+        return p
+
+def get_config():
+    cli_conf = OmegaConf.from_cli()
+    yaml_conf = OmegaConf.load(cli_conf.config)
+    conf = OmegaConf.merge(yaml_conf, cli_conf)
+    return conf
+
+def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]:
+    token = '<|endoftext|>'
+    if isinstance(obj, str):
+        return obj.replace(token, '')
+    if isinstance(obj, list):
+        return [s.replace(token, '') if isinstance(s, str) else s for s in obj]
+    raise TypeError("Expected str or list of str")
+
+
+if __name__ == "__main__":
+
+    config = get_config()
+    run = None
+    if config.experiment.function == "train":
+        existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None)
+        run_id = ensure_wandb_run_id(config)
+        resume_wandb_run = config.wandb.resume
+        if existing_run_id is None:
+            resume_wandb_run = False
+        wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
+        run = wandb.init(
+            project=config.experiment.project,
+            entity=config.wandb.get("entity", None),
+            id=run_id,
+            resume=resume_wandb_run,
+            config=wandb_config,
+            reinit=True,
+            name=config.experiment.project,
+        )
+
+    project_name = config.experiment.project
+    
+    if config.experiment.current_epoch == 1:
+        pretrained_model = config.model.pretrained_model
+    else:
+        pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name
+    
+    if config.experiment.function == "train":
+        shrink = config.training.shrink
+        dataset = config.dataset.train_dataset
+        outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}"
+        
+    elif config.experiment.function == "evaluation":
+        dataset = config.evaluation.eval_dataset
+        outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}"
+    
+    file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json"
+
+    with open(file_name, 'r') as f:
+        data = json.load(f)
+
+    index_list = []
+    extracted_output_list = []
+    response_length_list = []
+    for i in range(len(data)):
+        data[i]["correctness"] = []
+        response_length_list += data[i]["response_length"]
+        index_list += [i] * len(data[i]["extracted_output"])
+        extracted_output_list += remove_endoftext(data[i]["extracted_output"])
+
+    nest_asyncio.apply()
+    
+    async def get_correctness_all():
+        executor = ThreadPoolExecutor(max_workers=64)
+        tasks = []
+        
+        for p in range(len(index_list)):
+            pred = extracted_output_list[p]
+            idx = index_list[p]
+            
+            # 选择题正确性匹配逻辑
+            if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse":
+                gt = data[idx].get("response", "")
+                
+                async def mcq_check(p_str, g_str):
+                    if p_str in g_str or g_str in p_str:
+                        return True
+
+                    p_str = str(p_str).strip()
+                    g_str = str(g_str).strip()
+                    p_match = re.search(r'^([A-Z])[\.\s]', p_str)
+                    g_match = re.search(r'^([A-Z])[\.\s]', g_str)
+                    if p_match and g_match:
+                        return p_match.group(1) == g_match.group(1)
+                    return g_str in p_str
+                
+                tasks.append(mcq_check(pred, gt))
+            else:
+                gt = data[idx].get("ground_truth_answer", "")
+                tasks.append(math_utils_v.is_equal(pred, gt, executor))
+                
+        return await asyncio.gather(*tasks)
+
+    correctness_all = asyncio.run(get_correctness_all())
+    for p, ok in enumerate(correctness_all):
+        index_i = index_list[p]
+        data[index_i]["correctness"].append(bool(ok))
+
+    # =========================================================================
+    # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励
+    # =========================================================================
+    
+    # 1. 按照 data_id 分组映射索引
+    grouped_data = defaultdict(lambda: {
+        "forward_False": [], "backward_False": [], 
+        "forward_True": [],  "backward_True": []
+    })
+
+    for i in range(len(data)):
+        d_id = data[i].get("data_id")
+        if d_id is None:
+            continue
+        r_type = data[i].get("reversal_type", "forward")
+        is_shuf = str(data[i].get("is_shuffled", False))
+        grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i)
+
+    # 计算并应用群组奖励
+    all_temporal_bonuses = []  # 新增：记录所有的时序红利
+    all_unbiased_bonuses = []  # 新增：记录所有的无偏红利
+
+    # 2. 计算并应用群组奖励
+    for d_id, group in grouped_data.items():
+        
+        def get_acc(key):
+            if not group[key]: return 0.0
+            idx = group[key][0]
+            corr = data[idx]["correctness"]
+            return sum(corr) / len(corr) if len(corr) > 0 else 0.0
+
+        # 获取四组情况的正确率
+        acc_f_ord = get_acc("forward_False")
+        acc_f_shuf = get_acc("forward_True")
+        acc_b_ord = get_acc("backward_False")
+        acc_b_shuf = get_acc("backward_True")
+
+        # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3
+        f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0
+        b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0
+
+        # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3
+        unbiased_bonus = 0.0
+        if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1):
+            unbiased_bonus = 0.3
+
+        # 新增：将当前组的奖励记录下来用于统计
+        if group["forward_False"]:
+            all_temporal_bonuses.append(f_temporal_bonus)
+            all_unbiased_bonuses.append(unbiased_bonus)
+        if group["backward_False"]:
+            all_temporal_bonuses.append(b_temporal_bonus)
+            all_unbiased_bonuses.append(unbiased_bonus)
+
+        # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本
+        def apply_bonus(key, temporal_bonus, unbiased_bonus):
+            if not group[key]: return
+            idx = group[key][0]
+            raw_rewards = []
+            for is_correct in data[idx]["correctness"]:
+                if is_correct:
+                    # 基础分 1.0 + 时序红利 + 无偏红利
+                    raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus)
+                else:
+                    raw_rewards.append(0.0)
+            data[idx]["raw_rewards"] = raw_rewards
+
+        apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus)
+        apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus)
+
+    # =========================================================================
+
+    def z_score_normalize(lst):
+        mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0
+        std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0
+        if std == 0:
+            return [0 for _ in lst]
+        return [(x - mean) / std for x in lst]
+
+    final_data = []
+    ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据
+    
+    for i in range(len(data)):
+        # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习
+        if data[i].get("is_shuffled", False):
+            continue
+            
+        ordered_correctness.extend(data[i]["correctness"])
+
+        # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数
+        raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]])
+        lengths = data[i]["response_length"]
+        
+        # 长度惩罚
+        if config.reward.strict_len_check:
+            for j in range(len(lengths)):
+                if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5:
+                    raw_rewards[j] = 0.0
+                if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5:
+                    raw_rewards[j] = 0.0
+
+        # Z-Score 优势归一化
+        rewards = z_score_normalize(raw_rewards)
+        data[i]["rewards"] = rewards
+
+        if config.experiment.function == "train":
+            proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0
+            # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本
+            if proportion > 0.95 or proportion < 0.05:
+                continue
+
+            for j in range(len(rewards)):
+                data_i = {}
+                data_i["prompt"] = data[i]["prompt"]
+                data_i["reward"] = rewards[j]
+                data_i["response"] = data[i]["full_output"][j]
+                data_i["step_map"] = data[i]["step_map"][j]
+                
+                # 保留其他需要的上下文 ID
+                data_i["data_id"] = data[i].get("data_id")
+                data_i["reversal_type"] = data[i].get("reversal_type")
+                
+                if "image_token_ids" in data[i]:
+                    data_i["image_token_ids"] = data[i]["image_token_ids"]
+                
+                p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video")
+                if p_video_abs is not None:
+                    data_i["video_abs_path"] = p_video_abs
+                
+                p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image")
+                if p_image_abs is not None:
+                    data_i["image_abs_path"] = p_image_abs
+
+                final_data.append(data_i)
+
+        if config.experiment.function == "evaluation":
+            data[i]["step_map"] = []
+
+    if config.experiment.function == "train":
+        with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f:
+            json.dump(final_data, f, indent=2, ensure_ascii=False)
+
+
+    os.makedirs(os.path.dirname(file_name), exist_ok=True)
+    with open(file_name, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+
+
+    outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt"
+    os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True)
+    with open(outputs_result_name, "a") as f:
+        # Save + print
+        def save_and_print(text):
+            cprint("\n\n\n" + text, color="green")
+            f.write(text + "\n")
+        
+        # 只计算有序组（有效学习组）的准确率来作为监控指标
+        acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan")
+        avg_len = sum(response_length_list)/len(response_length_list)
+
+        # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值
+        avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0
+        avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0
+
+        output_text = f"train step: {config.experiment.current_epoch}  "
+        if config.experiment.function == "train":
+            if config.model.model_base != "sdar" and config.model.model_base != "trado":
+                output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy}  block_size: {config.rollout.block_size}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+            else:
+                output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy}  top_k: {config.rollout.top_k}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+        else:
+            if config.model.model_base != "sdar" and config.model.model_base != "trado":
+                output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy}  block_size: {config.evaluation.block_size}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+            else:
+                output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy}  top_k: {config.evaluation.top_k}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+        save_and_print(output_text)
+
+    if config.experiment.function == "train":
+        csv_file_path = f"../{project_name}/results/all_rewards.csv"
+        
+        # 如果是第1个 epoch/step，或者是新文件，就先写入表头
+        write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1
+        
+        with open(csv_file_path, "a", encoding="utf-8") as f_csv:
+            if write_header:
+                f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n")
+            
+            # 按行追加当前 step 的数据
+            f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n")
+
+        if run is not None:
+            wandb.log(
+                {
+                    "reward/acc": acc,
+                    "reward/t_grpo_bonus": avg_temp,
+                    "reward/unbiased_bonus": avg_unbias,
+                    "reward/avg_response_length": avg_len,
+                    "reward/train_step": config.experiment.current_epoch,
+                },
+                step=config.experiment.current_epoch,
+            )
+            wandb.finish()

reward/wandb/run-20260530_001305-fmmkx1a5/files/config.yaml

@@ -0,0 +1,281 @@
+_wandb:
+    value:
+        cli_version: 0.25.0
+        code_path: code/reward/rl_reward_video_reversal.py
+        e:
+            nbjlrwv1znq9ynwn7j97kr9f61plnzak:
+                args:
+                    - config=../configs/rl_lavida_video_reversal.yaml
+                    - wandb.run_id=fmmkx1a5
+                    - experiment.function=train
+                    - evaluation.block_size=None
+                    - evaluation.top_k=None
+                    - evaluation.remasking_strategy=None
+                    - experiment.current_epoch=4
+                codePath: reward/rl_reward_video_reversal.py
+                codePathLocal: rl_reward_video_reversal.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "806578724864"
+                        used: "625472454656"
+                email: shixi_runqing@staff.weibo.com
+                executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python
+                git:
+                    commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255
+                    remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git
+                gpu: NVIDIA H800
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-2775489f-18c8-2328-8829-c46238575379
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd
+                host: b1301b38c75f
+                memory:
+                    total: "2164113764352"
+                os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35
+                program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py
+                python: CPython 3.10.19
+                root: /data4/runqing/dLLM-RL/reward
+                startedAt: "2026-05-29T16:13:05.220431Z"
+                writerId: nbjlrwv1znq9ynwn7j97kr9f61plnzak
+        m: []
+        python_version: 3.10.19
+        t:
+            "1":
+                - 1
+                - 41
+            "2":
+                - 1
+                - 41
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 16
+                - 61
+                - 62
+            "4": 3.10.19
+            "5": 0.25.0
+            "10":
+                - 20
+            "12": 0.25.0
+            "13": linux-x86_64
+config:
+    value: ../configs/rl_lavida_video_reversal.yaml
+dataset.data_type:
+    value: video_reverse
+dataset.optimization_data:
+    value: rl_data
+dataset.train_dataset:
+    value: video_reversal_rl_train
+evaluation.batch_size:
+    value: 1
+evaluation.block_size:
+    value: None
+evaluation.data_type:
+    value: video_reverse
+evaluation.dynamic_threshold:
+    value: 0.9
+evaluation.eval_dataset:
+    value: video_reversal_rl_eval
+evaluation.further_horizon:
+    value: 128
+evaluation.if_eval:
+    value: false
+evaluation.max_gen_length:
+    value: 128
+evaluation.num_response_per_task:
+    value: 3
+evaluation.remasking_strategy:
+    value: None
+evaluation.remasking_strategy.0:
+    value: low_confidence_static
+evaluation.remasking_strategy.1:
+    value: low_confidence_dynamic
+evaluation.steps:
+    value: 128
+evaluation.target:
+    value: confidence
+evaluation.temperature:
+    value: 0.1
+evaluation.top_k:
+    value: None
+evaluation.use_cache:
+    value: true
+execute.num_chunk:
+    value: 128
+experiment.current_epoch:
+    value: 4
+experiment.deepspeed_file:
+    value: 1_node_8_gpus_deepspeed_zero2
+experiment.eval_every:
+    value: 5
+experiment.function:
+    value: train
+experiment.logging_dir:
+    value: rl_lavida_video_reversal/logs
+experiment.node_index:
+    value: 0
+experiment.num_node:
+    value: 1
+experiment.project:
+    value: rl_lavida_video_reversal
+experiment.save_every:
+    value: 10
+experiment.start_from_scratch:
+    value: true
+experiment.total_step:
+    value: 30
+lr_scheduler.params.learning_rate:
+    value: 1e-06
+lr_scheduler.params.min_lr_scale:
+    value: 1
+lr_scheduler.params.warmup_steps:
+    value: 0
+lr_scheduler.scheduler:
+    value: cosine
+model.image_resolution:
+    value: 512
+model.model_base:
+    value: lavida
+model.optimized_name:
+    value: optimized
+model.pretrained_model:
+    value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized
+model.vq_model_path:
+    value: /data4/runqing/hf/magvitv2
+optimizer.name:
+    value: adamw
+optimizer.params.beta1:
+    value: 0.9
+optimizer.params.beta2:
+    value: 0.999
+optimizer.params.epsilon:
+    value: 1e-08
+optimizer.params.learning_rate:
+    value: 1e-06
+optimizer.params.scale_lr:
+    value: false
+optimizer.params.weight_decay:
+    value: 0
+reward.answer_must_in_box:
+    value: true
+reward.strict_len_check:
+    value: false
+rollout.batch_size:
+    value: 1
+rollout.block_size:
+    value: 32
+rollout.dynamic_threshold:
+    value: 0.9
+rollout.further_horizon:
+    value: 128
+rollout.max_gen_length:
+    value: 128
+rollout.num_response_per_task:
+    value: 8
+rollout.num_task_per_step:
+    value: 128
+rollout.remasking_strategy:
+    value: low_confidence_dynamic
+rollout.steps:
+    value: 128
+rollout.target:
+    value: confidence
+rollout.temperature:
+    value: 0.8
+rollout.use_cache:
+    value: true
+system.HF_HOME:
+    value: null
+system.HTTP_PROXY:
+    value: null
+system.base_dir:
+    value: /data4/runqing/dLLM-RL
+system.env_name:
+    value: /data8/runqing/miniconda/envs/dllm-rl
+system.envs_dir:
+    value: /data8/runqing/miniconda/envs/dllm-rl
+training.batch_size_lm:
+    value: 1
+training.beta:
+    value: 0.01
+training.enable_tf32:
+    value: true
+training.eps:
+    value: 0.2
+training.gradient_accumulation_steps:
+    value: 2
+training.lower_p:
+    value: 0.35
+training.mask_times_per_sample:
+    value: 2
+training.max_gen_length:
+    value: 128
+training.max_grad_norm:
+    value: 1
+training.max_prompt_len:
+    value: 512
+training.method:
+    value: random_masking
+training.mixed_precision:
+    value: bf16
+training.num_train_epochs:
+    value: 1
+training.post_num:
+    value: 1
+training.seed:
+    value: 10086
+training.shrink:
+    value: 8
+training.upper_p:
+    value: 0.9
+training.use_kl_estimator_k3:
+    value: true
+wandb.entity:
+    value: null
+wandb.resume:
+    value: auto
+wandb.run_id:
+    value: fmmkx1a5

reward/wandb/run-20260530_001305-fmmkx1a5/files/output.log

@@ -0,0 +1,4 @@
+
+
+
+train step: 4  remasking_strategy: low_confidence_dynamic  block_size: 32  acc: 0.4590  avg length: 126.43

reward/wandb/run-20260530_001305-fmmkx1a5/files/requirements.txt

@@ -0,0 +1,157 @@
+transformers==4.52.4
+nvidia-nvtx-cu12==12.4.127
+deepspeed==0.18.6
+typing-inspection==0.4.2
+scipy==1.15.3
+MarkupSafe==3.0.3
+nvidia-cuda-runtime-cu12==12.4.127
+detoxify==0.5.2
+yarl==1.22.0
+tornado==6.5.4
+nvidia-curand-cu12==10.3.5.147
+nest-asyncio==1.6.0
+safetensors==0.7.0
+async-timeout==5.0.1
+typing_extensions==4.15.0
+asttokens==3.0.1
+cuda-python==13.1.1
+executing==2.2.1
+nvidia-cudnn-frontend==1.18.0
+comm==0.2.3
+tokenizers==0.21.4
+six==1.17.0
+Pygments==2.19.2
+nvidia-cusparselt-cu12==0.6.2
+apache-tvm-ffi==0.1.8.post2
+flash_attn==2.7.4.post1
+tqdm==4.67.3
+tensorboard-data-server==0.7.2
+annotated-types==0.7.0
+math-verify==0.9.0
+idna==3.11
+regex==2026.1.15
+nvidia-cutlass-dsl-libs-base==4.4.0
+flashinfer-python==0.6.3
+click==8.3.1
+pytz==2025.2
+wheel==0.46.3
+ipykernel==7.2.0
+stack-data==0.6.3
+absl-py==2.4.0
+wandb==0.25.0
+pandas==2.3.3
+termcolor==3.3.0
+sympy==1.13.1
+joblib==1.5.3
+prompt_toolkit==3.0.52
+attrs==25.4.0
+nvidia-cusolver-cu12==11.6.1.9
+xxhash==3.6.0
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+frozenlist==1.8.0
+nvidia-ml-py==13.590.48
+multiprocess==0.70.18
+decorator==5.2.1
+cuda-pathfinder==1.3.4
+multidict==6.7.1
+pure_eval==0.2.3
+nvidia-cufft-cu12==11.2.1.3
+omegaconf==2.3.0
+PyYAML==6.0.3
+debugpy==1.8.20
+psutil==7.2.2
+ninja==1.13.0
+nvidia-cublas-cu12==12.4.5.8
+triton==3.2.0
+antlr4-python3-runtime==4.9.3
+peft==0.18.1
+aiosignal==1.4.0
+accelerate==1.12.0
+liger_kernel==0.7.0
+anyio==4.12.1
+decord==0.6.0
+h11==0.16.0
+urllib3==2.6.3
+wcwidth==0.6.0
+networkx==3.4.2
+huggingface_hub==0.36.2
+tzdata==2025.3
+xformers==0.0.29.post3
+tabulate==0.9.0
+certifi==2026.1.4
+tensorboard==2.20.0
+matplotlib-inline==0.2.1
+pip==26.0.1
+smmap==5.0.2
+nvidia-cuda-cupti-cu12==12.4.127
+Markdown==3.10.2
+nvidia-nccl-cu12==2.21.5
+aiohttp==3.13.3
+scikit-learn==1.7.2
+nvidia-cutlass-dsl==4.4.0
+setuptools==82.0.0
+jupyter_core==5.9.1
+asyncio==4.0.0
+einops==0.8.2
+pydantic==2.12.5
+av==16.1.0
+protobuf==6.33.5
+evaluate==0.4.6
+Jinja2==3.1.6
+exceptiongroup==1.3.1
+ipython==8.38.0
+sentry-sdk==2.53.0
+httpx==0.28.1
+msgpack==1.1.2
+platformdirs==4.9.2
+hjson==3.1.0
+torchvision==0.21.0
+parso==0.8.6
+cuda-bindings==13.1.1
+pexpect==4.9.0
+threadpoolctl==3.6.0
+pyarrow==23.0.1
+jedi==0.19.2
+nvidia-cuda-nvrtc-cu12==12.4.127
+Werkzeug==3.1.5
+fsspec==2025.10.0
+pydantic_core==2.41.5
+GitPython==3.1.46
+propcache==0.4.1
+mpmath==1.3.0
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-cusparse-cu12==12.3.1.170
+hf-xet==1.2.0
+charset-normalizer==3.4.4
+dill==0.4.0
+datasets==4.5.0
+py-cpuinfo==9.0.0
+sentencepiece==0.2.1
+latex2sympy2_extended==1.11.0
+jupyter_client==8.8.0
+pyzmq==27.1.0
+numpy==2.2.6
+gitdb==4.0.12
+grpcio==1.78.0
+packaging==26.0
+requests==2.32.5
+nvidia-cudnn-cu12==9.1.0.70
+httpcore==1.0.9
+traitlets==5.14.3
+filelock==3.24.2
+ptyprocess==0.7.0
+pillow==12.1.1
+torch==2.6.0
+more-itertools==10.8.0
+wheel==0.46.3
+jaraco.text==4.0.0
+zipp==3.23.0
+autocommand==2.2.2
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+tomli==2.4.0
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+platformdirs==4.4.0
+packaging==26.0

reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-metadata.json

@@ -0,0 +1,98 @@
+{
+  "os":  "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.19",
+  "startedAt":  "2026-05-29T16:13:05.220431Z",
+  "args":  [
+    "config=../configs/rl_lavida_video_reversal.yaml",
+    "wandb.run_id=fmmkx1a5",
+    "experiment.function=train",
+    "evaluation.block_size=None",
+    "evaluation.top_k=None",
+    "evaluation.remasking_strategy=None",
+    "experiment.current_epoch=4"
+  ],
+  "program":  "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py",
+  "codePath":  "reward/rl_reward_video_reversal.py",
+  "codePathLocal":  "rl_reward_video_reversal.py",
+  "git":  {
+    "remote":  "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
+    "commit":  "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
+  },
+  "email":  "shixi_runqing@staff.weibo.com",
+  "root":  "/data4/runqing/dLLM-RL/reward",
+  "host":  "b1301b38c75f",
+  "executable":  "/data8/runqing/miniconda/envs/dllm-rl/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H800",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "806578724864",
+      "used":  "625472454656"
+    }
+  },
+  "memory":  {
+    "total":  "2164113764352"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2775489f-18c8-2328-8829-c46238575379"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
+    }
+  ],
+  "cudaVersion":  "12.8",
+  "writerId":  "nbjlrwv1znq9ynwn7j97kr9f61plnzak"
+}

reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime":2202,"reward/t_grpo_bonus":0.2718750000000003,"reward/unbiased_bonus":0.09374999999999996,"train/learning_rate":7.300325188655752e-07,"reward/acc":0.396484375,"_step":22,"train/loss":0.3783190548419952,"reward/train_step":1,"reward/avg_response_length":124.568359375,"_wandb":{"runtime":2202},"_timestamp":1.7800614303455548e+09}

reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug-core.log

@@ -0,0 +1,18 @@
+{"time":"2026-05-30T00:13:05.286065679+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpp62euswa/port-1751242.txt","pid":1751242,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-05-30T00:13:05.287189973+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1751242}
+{"time":"2026-05-30T00:13:05.287151511+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1751242-1751409-2046619474/socket","Net":"unix"}}
+{"time":"2026-05-30T00:13:05.468769376+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-05-30T00:13:05.477564719+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T00:13:05.780034643+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T00:13:06.242278206+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"k7igfwpnqhw4"}
+{"time":"2026-05-30T00:13:06.425328395+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"k7igfwpnqhw4"}
+{"time":"2026-05-30T00:13:06.42595584+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T00:13:06.42694024+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T00:13:06.426952049+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-05-30T00:13:06.426964821+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-05-30T00:13:06.426975472+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-05-30T00:13:06.426985269+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-05-30T00:13:06.427015094+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-05-30T00:13:06.427038141+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-05-30T00:13:06.427156462+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1751242-1751409-2046619474/socket","Net":"unix"}}
+{"time":"2026-05-30T00:13:06.427212116+08:00","level":"INFO","msg":"server is closed"}

reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug-internal.log

@@ -0,0 +1,28 @@
+{"time":"2026-05-30T00:13:05.477770765+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
+{"time":"2026-05-30T00:13:05.779450034+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
+{"time":"2026-05-30T00:13:05.779652585+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"}
+{"time":"2026-05-30T00:13:05.779744219+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T00:13:05.780014388+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"}
+{"time":"2026-05-30T00:13:05.780055478+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T00:13:05.780055506+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T00:13:05.782697905+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
+{"time":"2026-05-30T00:13:06.028313692+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-30T00:13:06.2385663+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":4,"current":23}
+{"time":"2026-05-30T00:13:06.368370958+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8578fc7b87857bdd751003ba69247abe51fa215fdca5c83876ba57e01529f06f"}
+{"time":"2026-05-30T00:13:06.368497869+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log</Resource><RequestId>18B415501297BED1</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=8578fc7b87857bdd751003ba69247abe51fa215fdca5c83876ba57e01529f06f, Size: 110}"}
+{"time":"2026-05-30T00:13:06.368902337+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=74595d174d88f5f42545e11c808630835e87001e8a5296f7c2c718f2d4995bdc"}
+{"time":"2026-05-30T00:13:06.369039843+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml</Resource><RequestId>18B415501291CB61</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=74595d174d88f5f42545e11c808630835e87001e8a5296f7c2c718f2d4995bdc, Size: 7994}"}
+{"time":"2026-05-30T00:13:06.369639945+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a2d0a58bb258141ad482f96245ac2182bdfbbdab1e2fa96a04db7ecbc47d2843"}
+{"time":"2026-05-30T00:13:06.369798939+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json</Resource><RequestId>18B4155012169184</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=a2d0a58bb258141ad482f96245ac2182bdfbbdab1e2fa96a04db7ecbc47d2843, Size: 2849}"}
+{"time":"2026-05-30T00:13:06.370661611+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=99480e690e5934203d2d0ba772e2d0ce0a39c37c4d8b6b6d9db3c4482f0b9fc1"}
+{"time":"2026-05-30T00:13:06.370783078+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json</Resource><RequestId>18B41550110447D3</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=99480e690e5934203d2d0ba772e2d0ce0a39c37c4d8b6b6d9db3c4482f0b9fc1, Size: 341}"}
+{"time":"2026-05-30T00:13:06.370857593+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=932abcfbee1a259382384175d32ca10d553bb10e5e18f491f305f95f6f890ab1"}
+{"time":"2026-05-30T00:13:06.370975128+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt</Resource><RequestId>18B41550122B1FAB</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=932abcfbee1a259382384175d32ca10d553bb10e5e18f491f305f95f6f890ab1, Size: 2982}"}
+{"time":"2026-05-30T00:13:06.378523807+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce774c4d207911537ae5971450542027f4d47ab73f9f0e5a233371ed80539cb7"}
+{"time":"2026-05-30T00:13:06.378678161+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py</Resource><RequestId>18B4155010FA7E6E</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T161306Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=ce774c4d207911537ae5971450542027f4d47ab73f9f0e5a233371ed80539cb7, Size: 14790}"}
+{"time":"2026-05-30T00:13:06.378940351+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-30T00:13:06.421106068+08:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-05-30T00:13:06.425978529+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"}
+{"time":"2026-05-30T00:13:06.425995378+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T00:13:06.426060973+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T00:13:06.426080026+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"}

reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug.log

@@ -0,0 +1,25 @@
+2026-05-30 00:13:05,223 INFO    MainThread:1751242 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
+2026-05-30 00:13:05,223 INFO    MainThread:1751242 [wandb_setup.py:_flush():81] Configure stats pid to 1751242
+2026-05-30 00:13:05,223 INFO    MainThread:1751242 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-30 00:13:05,223 INFO    MainThread:1751242 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug.log
+2026-05-30 00:13:05,223 INFO    MainThread:1751242 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_001305-fmmkx1a5/logs/debug-internal.log
+2026-05-30 00:13:05,223 INFO    MainThread:1751242 [wandb_init.py:init():844] calling init triggers
+2026-05-30 00:13:05,223 INFO    MainThread:1751242 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 4, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}}
+2026-05-30 00:13:05,223 INFO    MainThread:1751242 [wandb_init.py:init():892] starting backend
+2026-05-30 00:13:05,469 INFO    MainThread:1751242 [wandb_init.py:init():895] sending inform_init request
+2026-05-30 00:13:05,474 INFO    MainThread:1751242 [wandb_init.py:init():903] backend started and connected
+2026-05-30 00:13:05,475 INFO    MainThread:1751242 [wandb_init.py:init():973] updated telemetry
+2026-05-30 00:13:05,479 INFO    MainThread:1751242 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-30 00:13:06,023 INFO    MainThread:1751242 [wandb_init.py:init():1037] run resumed
+2026-05-30 00:13:06,026 INFO    MainThread:1751242 [wandb_init.py:init():1042] starting run threads in backend
+2026-05-30 00:13:06,116 INFO    MainThread:1751242 [wandb_run.py:_console_start():2524] atexit reg
+2026-05-30 00:13:06,116 INFO    MainThread:1751242 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-05-30 00:13:06,116 INFO    MainThread:1751242 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-05-30 00:13:06,116 INFO    MainThread:1751242 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-05-30 00:13:06,120 INFO    MainThread:1751242 [wandb_init.py:init():1082] run started, returning control to user process
+2026-05-30 00:13:06,238 INFO    MainThread:1751242 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5
+2026-05-30 00:13:06,238 INFO    MainThread:1751242 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
+2026-05-30 00:13:06,239 INFO    MainThread:1751242 [wandb_run.py:_restore():2472] restore
+2026-05-30 00:13:06,239 INFO    MainThread:1751242 [wandb_run.py:_restore():2478] restore done
+2026-05-30 00:13:06,425 INFO    MainThread:1751242 [wandb_run.py:_footer_sync_info():3868] logging synced files

reward/wandb/run-20260530_022315-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py

@@ -0,0 +1,339 @@
+import json
+import math_utils_v
+import nest_asyncio
+from concurrent.futures import ThreadPoolExecutor
+import asyncio
+import re
+from termcolor import cprint
+from omegaconf import MISSING
+from omegaconf import DictConfig, ListConfig, OmegaConf
+from typing import List, Union
+import os
+from collections import defaultdict
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, REPO_ROOT)
+
+import wandb
+from train.utils import flatten_omega_conf, ensure_wandb_run_id
+
+def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
+    p = sample.get(f"{media_type}_abs_path")
+    if p is None:
+        p = sample.get(media_type) or sample.get(f"{media_type}_path")
+        if p is None:
+            return None
+        if not os.path.isabs(p) and dataset_root:
+            p = os.path.join(dataset_root, p)
+    try:
+        return os.path.abspath(p)
+    except Exception:
+        return p
+
+def get_config():
+    cli_conf = OmegaConf.from_cli()
+    yaml_conf = OmegaConf.load(cli_conf.config)
+    conf = OmegaConf.merge(yaml_conf, cli_conf)
+    return conf
+
+def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]:
+    token = '<|endoftext|>'
+    if isinstance(obj, str):
+        return obj.replace(token, '')
+    if isinstance(obj, list):
+        return [s.replace(token, '') if isinstance(s, str) else s for s in obj]
+    raise TypeError("Expected str or list of str")
+
+
+if __name__ == "__main__":
+
+    config = get_config()
+    run = None
+    if config.experiment.function == "train":
+        existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None)
+        run_id = ensure_wandb_run_id(config)
+        resume_wandb_run = config.wandb.resume
+        if existing_run_id is None:
+            resume_wandb_run = False
+        wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
+        run = wandb.init(
+            project=config.experiment.project,
+            entity=config.wandb.get("entity", None),
+            id=run_id,
+            resume=resume_wandb_run,
+            config=wandb_config,
+            reinit=True,
+            name=config.experiment.project,
+        )
+
+    project_name = config.experiment.project
+    
+    if config.experiment.current_epoch == 1:
+        pretrained_model = config.model.pretrained_model
+    else:
+        pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name
+    
+    if config.experiment.function == "train":
+        shrink = config.training.shrink
+        dataset = config.dataset.train_dataset
+        outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}"
+        
+    elif config.experiment.function == "evaluation":
+        dataset = config.evaluation.eval_dataset
+        outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}"
+    
+    file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json"
+
+    with open(file_name, 'r') as f:
+        data = json.load(f)
+
+    index_list = []
+    extracted_output_list = []
+    response_length_list = []
+    for i in range(len(data)):
+        data[i]["correctness"] = []
+        response_length_list += data[i]["response_length"]
+        index_list += [i] * len(data[i]["extracted_output"])
+        extracted_output_list += remove_endoftext(data[i]["extracted_output"])
+
+    nest_asyncio.apply()
+    
+    async def get_correctness_all():
+        executor = ThreadPoolExecutor(max_workers=64)
+        tasks = []
+        
+        for p in range(len(index_list)):
+            pred = extracted_output_list[p]
+            idx = index_list[p]
+            
+            # 选择题正确性匹配逻辑
+            if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse":
+                gt = data[idx].get("response", "")
+                
+                async def mcq_check(p_str, g_str):
+                    if p_str in g_str or g_str in p_str:
+                        return True
+
+                    p_str = str(p_str).strip()
+                    g_str = str(g_str).strip()
+                    p_match = re.search(r'^([A-Z])[\.\s]', p_str)
+                    g_match = re.search(r'^([A-Z])[\.\s]', g_str)
+                    if p_match and g_match:
+                        return p_match.group(1) == g_match.group(1)
+                    return g_str in p_str
+                
+                tasks.append(mcq_check(pred, gt))
+            else:
+                gt = data[idx].get("ground_truth_answer", "")
+                tasks.append(math_utils_v.is_equal(pred, gt, executor))
+                
+        return await asyncio.gather(*tasks)
+
+    correctness_all = asyncio.run(get_correctness_all())
+    for p, ok in enumerate(correctness_all):
+        index_i = index_list[p]
+        data[index_i]["correctness"].append(bool(ok))
+
+    # =========================================================================
+    # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励
+    # =========================================================================
+    
+    # 1. 按照 data_id 分组映射索引
+    grouped_data = defaultdict(lambda: {
+        "forward_False": [], "backward_False": [], 
+        "forward_True": [],  "backward_True": []
+    })
+
+    for i in range(len(data)):
+        d_id = data[i].get("data_id")
+        if d_id is None:
+            continue
+        r_type = data[i].get("reversal_type", "forward")
+        is_shuf = str(data[i].get("is_shuffled", False))
+        grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i)
+
+    # 计算并应用群组奖励
+    all_temporal_bonuses = []  # 新增：记录所有的时序红利
+    all_unbiased_bonuses = []  # 新增：记录所有的无偏红利
+
+    # 2. 计算并应用群组奖励
+    for d_id, group in grouped_data.items():
+        
+        def get_acc(key):
+            if not group[key]: return 0.0
+            idx = group[key][0]
+            corr = data[idx]["correctness"]
+            return sum(corr) / len(corr) if len(corr) > 0 else 0.0
+
+        # 获取四组情况的正确率
+        acc_f_ord = get_acc("forward_False")
+        acc_f_shuf = get_acc("forward_True")
+        acc_b_ord = get_acc("backward_False")
+        acc_b_shuf = get_acc("backward_True")
+
+        # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3
+        f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0
+        b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0
+
+        # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3
+        unbiased_bonus = 0.0
+        if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1):
+            unbiased_bonus = 0.3
+
+        # 新增：将当前组的奖励记录下来用于统计
+        if group["forward_False"]:
+            all_temporal_bonuses.append(f_temporal_bonus)
+            all_unbiased_bonuses.append(unbiased_bonus)
+        if group["backward_False"]:
+            all_temporal_bonuses.append(b_temporal_bonus)
+            all_unbiased_bonuses.append(unbiased_bonus)
+
+        # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本
+        def apply_bonus(key, temporal_bonus, unbiased_bonus):
+            if not group[key]: return
+            idx = group[key][0]
+            raw_rewards = []
+            for is_correct in data[idx]["correctness"]:
+                if is_correct:
+                    # 基础分 1.0 + 时序红利 + 无偏红利
+                    raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus)
+                else:
+                    raw_rewards.append(0.0)
+            data[idx]["raw_rewards"] = raw_rewards
+
+        apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus)
+        apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus)
+
+    # =========================================================================
+
+    def z_score_normalize(lst):
+        mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0
+        std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0
+        if std == 0:
+            return [0 for _ in lst]
+        return [(x - mean) / std for x in lst]
+
+    final_data = []
+    ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据
+    
+    for i in range(len(data)):
+        # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习
+        if data[i].get("is_shuffled", False):
+            continue
+            
+        ordered_correctness.extend(data[i]["correctness"])
+
+        # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数
+        raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]])
+        lengths = data[i]["response_length"]
+        
+        # 长度惩罚
+        if config.reward.strict_len_check:
+            for j in range(len(lengths)):
+                if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5:
+                    raw_rewards[j] = 0.0
+                if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5:
+                    raw_rewards[j] = 0.0
+
+        # Z-Score 优势归一化
+        rewards = z_score_normalize(raw_rewards)
+        data[i]["rewards"] = rewards
+
+        if config.experiment.function == "train":
+            proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0
+            # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本
+            if proportion > 0.95 or proportion < 0.05:
+                continue
+
+            for j in range(len(rewards)):
+                data_i = {}
+                data_i["prompt"] = data[i]["prompt"]
+                data_i["reward"] = rewards[j]
+                data_i["response"] = data[i]["full_output"][j]
+                data_i["step_map"] = data[i]["step_map"][j]
+                
+                # 保留其他需要的上下文 ID
+                data_i["data_id"] = data[i].get("data_id")
+                data_i["reversal_type"] = data[i].get("reversal_type")
+                
+                if "image_token_ids" in data[i]:
+                    data_i["image_token_ids"] = data[i]["image_token_ids"]
+                
+                p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video")
+                if p_video_abs is not None:
+                    data_i["video_abs_path"] = p_video_abs
+                
+                p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image")
+                if p_image_abs is not None:
+                    data_i["image_abs_path"] = p_image_abs
+
+                final_data.append(data_i)
+
+        if config.experiment.function == "evaluation":
+            data[i]["step_map"] = []
+
+    if config.experiment.function == "train":
+        with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f:
+            json.dump(final_data, f, indent=2, ensure_ascii=False)
+
+
+    os.makedirs(os.path.dirname(file_name), exist_ok=True)
+    with open(file_name, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+
+
+    outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt"
+    os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True)
+    with open(outputs_result_name, "a") as f:
+        # Save + print
+        def save_and_print(text):
+            cprint("\n\n\n" + text, color="green")
+            f.write(text + "\n")
+        
+        # 只计算有序组（有效学习组）的准确率来作为监控指标
+        acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan")
+        avg_len = sum(response_length_list)/len(response_length_list)
+
+        # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值
+        avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0
+        avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0
+
+        output_text = f"train step: {config.experiment.current_epoch}  "
+        if config.experiment.function == "train":
+            if config.model.model_base != "sdar" and config.model.model_base != "trado":
+                output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy}  block_size: {config.rollout.block_size}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+            else:
+                output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy}  top_k: {config.rollout.top_k}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+        else:
+            if config.model.model_base != "sdar" and config.model.model_base != "trado":
+                output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy}  block_size: {config.evaluation.block_size}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+            else:
+                output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy}  top_k: {config.evaluation.top_k}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+        save_and_print(output_text)
+
+    if config.experiment.function == "train":
+        csv_file_path = f"../{project_name}/results/all_rewards.csv"
+        
+        # 如果是第1个 epoch/step，或者是新文件，就先写入表头
+        write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1
+        
+        with open(csv_file_path, "a", encoding="utf-8") as f_csv:
+            if write_header:
+                f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n")
+            
+            # 按行追加当前 step 的数据
+            f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n")
+
+        if run is not None:
+            wandb.log(
+                {
+                    "reward/acc": acc,
+                    "reward/t_grpo_bonus": avg_temp,
+                    "reward/unbiased_bonus": avg_unbias,
+                    "reward/avg_response_length": avg_len,
+                    "reward/train_step": config.experiment.current_epoch,
+                },
+                step=config.experiment.current_epoch,
+            )
+            wandb.finish()

reward/wandb/run-20260530_022315-fmmkx1a5/files/config.yaml

@@ -0,0 +1,281 @@
+_wandb:
+    value:
+        cli_version: 0.25.0
+        code_path: code/reward/rl_reward_video_reversal.py
+        e:
+            rit67u7afan4qam30286z3mhxid958av:
+                args:
+                    - config=../configs/rl_lavida_video_reversal.yaml
+                    - wandb.run_id=fmmkx1a5
+                    - experiment.function=train
+                    - evaluation.block_size=None
+                    - evaluation.top_k=None
+                    - evaluation.remasking_strategy=None
+                    - experiment.current_epoch=6
+                codePath: reward/rl_reward_video_reversal.py
+                codePathLocal: rl_reward_video_reversal.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "806578724864"
+                        used: "625479712768"
+                email: shixi_runqing@staff.weibo.com
+                executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python
+                git:
+                    commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255
+                    remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git
+                gpu: NVIDIA H800
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-2775489f-18c8-2328-8829-c46238575379
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd
+                host: b1301b38c75f
+                memory:
+                    total: "2164113764352"
+                os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35
+                program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py
+                python: CPython 3.10.19
+                root: /data4/runqing/dLLM-RL/reward
+                startedAt: "2026-05-29T18:23:15.496253Z"
+                writerId: rit67u7afan4qam30286z3mhxid958av
+        m: []
+        python_version: 3.10.19
+        t:
+            "1":
+                - 1
+                - 41
+            "2":
+                - 1
+                - 41
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 16
+                - 61
+                - 62
+            "4": 3.10.19
+            "5": 0.25.0
+            "10":
+                - 20
+            "12": 0.25.0
+            "13": linux-x86_64
+config:
+    value: ../configs/rl_lavida_video_reversal.yaml
+dataset.data_type:
+    value: video_reverse
+dataset.optimization_data:
+    value: rl_data
+dataset.train_dataset:
+    value: video_reversal_rl_train
+evaluation.batch_size:
+    value: 1
+evaluation.block_size:
+    value: None
+evaluation.data_type:
+    value: video_reverse
+evaluation.dynamic_threshold:
+    value: 0.9
+evaluation.eval_dataset:
+    value: video_reversal_rl_eval
+evaluation.further_horizon:
+    value: 128
+evaluation.if_eval:
+    value: false
+evaluation.max_gen_length:
+    value: 128
+evaluation.num_response_per_task:
+    value: 3
+evaluation.remasking_strategy:
+    value: None
+evaluation.remasking_strategy.0:
+    value: low_confidence_static
+evaluation.remasking_strategy.1:
+    value: low_confidence_dynamic
+evaluation.steps:
+    value: 128
+evaluation.target:
+    value: confidence
+evaluation.temperature:
+    value: 0.1
+evaluation.top_k:
+    value: None
+evaluation.use_cache:
+    value: true
+execute.num_chunk:
+    value: 128
+experiment.current_epoch:
+    value: 6
+experiment.deepspeed_file:
+    value: 1_node_8_gpus_deepspeed_zero2
+experiment.eval_every:
+    value: 5
+experiment.function:
+    value: train
+experiment.logging_dir:
+    value: rl_lavida_video_reversal/logs
+experiment.node_index:
+    value: 0
+experiment.num_node:
+    value: 1
+experiment.project:
+    value: rl_lavida_video_reversal
+experiment.save_every:
+    value: 10
+experiment.start_from_scratch:
+    value: true
+experiment.total_step:
+    value: 30
+lr_scheduler.params.learning_rate:
+    value: 1e-06
+lr_scheduler.params.min_lr_scale:
+    value: 1
+lr_scheduler.params.warmup_steps:
+    value: 0
+lr_scheduler.scheduler:
+    value: cosine
+model.image_resolution:
+    value: 512
+model.model_base:
+    value: lavida
+model.optimized_name:
+    value: optimized
+model.pretrained_model:
+    value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized
+model.vq_model_path:
+    value: /data4/runqing/hf/magvitv2
+optimizer.name:
+    value: adamw
+optimizer.params.beta1:
+    value: 0.9
+optimizer.params.beta2:
+    value: 0.999
+optimizer.params.epsilon:
+    value: 1e-08
+optimizer.params.learning_rate:
+    value: 1e-06
+optimizer.params.scale_lr:
+    value: false
+optimizer.params.weight_decay:
+    value: 0
+reward.answer_must_in_box:
+    value: true
+reward.strict_len_check:
+    value: false
+rollout.batch_size:
+    value: 1
+rollout.block_size:
+    value: 32
+rollout.dynamic_threshold:
+    value: 0.9
+rollout.further_horizon:
+    value: 128
+rollout.max_gen_length:
+    value: 128
+rollout.num_response_per_task:
+    value: 8
+rollout.num_task_per_step:
+    value: 128
+rollout.remasking_strategy:
+    value: low_confidence_dynamic
+rollout.steps:
+    value: 128
+rollout.target:
+    value: confidence
+rollout.temperature:
+    value: 0.8
+rollout.use_cache:
+    value: true
+system.HF_HOME:
+    value: null
+system.HTTP_PROXY:
+    value: null
+system.base_dir:
+    value: /data4/runqing/dLLM-RL
+system.env_name:
+    value: /data8/runqing/miniconda/envs/dllm-rl
+system.envs_dir:
+    value: /data8/runqing/miniconda/envs/dllm-rl
+training.batch_size_lm:
+    value: 1
+training.beta:
+    value: 0.01
+training.enable_tf32:
+    value: true
+training.eps:
+    value: 0.2
+training.gradient_accumulation_steps:
+    value: 2
+training.lower_p:
+    value: 0.35
+training.mask_times_per_sample:
+    value: 2
+training.max_gen_length:
+    value: 128
+training.max_grad_norm:
+    value: 1
+training.max_prompt_len:
+    value: 512
+training.method:
+    value: random_masking
+training.mixed_precision:
+    value: bf16
+training.num_train_epochs:
+    value: 1
+training.post_num:
+    value: 1
+training.seed:
+    value: 10086
+training.shrink:
+    value: 8
+training.upper_p:
+    value: 0.9
+training.use_kl_estimator_k3:
+    value: true
+wandb.entity:
+    value: null
+wandb.resume:
+    value: auto
+wandb.run_id:
+    value: fmmkx1a5

reward/wandb/run-20260530_022315-fmmkx1a5/files/output.log

@@ -0,0 +1,4 @@
+
+
+
+train step: 6  remasking_strategy: low_confidence_dynamic  block_size: 32  acc: 0.3926  avg length: 126.23

reward/wandb/run-20260530_022315-fmmkx1a5/files/requirements.txt

@@ -0,0 +1,157 @@
+transformers==4.52.4
+nvidia-nvtx-cu12==12.4.127
+deepspeed==0.18.6
+typing-inspection==0.4.2
+scipy==1.15.3
+MarkupSafe==3.0.3
+nvidia-cuda-runtime-cu12==12.4.127
+detoxify==0.5.2
+yarl==1.22.0
+tornado==6.5.4
+nvidia-curand-cu12==10.3.5.147
+nest-asyncio==1.6.0
+safetensors==0.7.0
+async-timeout==5.0.1
+typing_extensions==4.15.0
+asttokens==3.0.1
+cuda-python==13.1.1
+executing==2.2.1
+nvidia-cudnn-frontend==1.18.0
+comm==0.2.3
+tokenizers==0.21.4
+six==1.17.0
+Pygments==2.19.2
+nvidia-cusparselt-cu12==0.6.2
+apache-tvm-ffi==0.1.8.post2
+flash_attn==2.7.4.post1
+tqdm==4.67.3
+tensorboard-data-server==0.7.2
+annotated-types==0.7.0
+math-verify==0.9.0
+idna==3.11
+regex==2026.1.15
+nvidia-cutlass-dsl-libs-base==4.4.0
+flashinfer-python==0.6.3
+click==8.3.1
+pytz==2025.2
+wheel==0.46.3
+ipykernel==7.2.0
+stack-data==0.6.3
+absl-py==2.4.0
+wandb==0.25.0
+pandas==2.3.3
+termcolor==3.3.0
+sympy==1.13.1
+joblib==1.5.3
+prompt_toolkit==3.0.52
+attrs==25.4.0
+nvidia-cusolver-cu12==11.6.1.9
+xxhash==3.6.0
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+frozenlist==1.8.0
+nvidia-ml-py==13.590.48
+multiprocess==0.70.18
+decorator==5.2.1
+cuda-pathfinder==1.3.4
+multidict==6.7.1
+pure_eval==0.2.3
+nvidia-cufft-cu12==11.2.1.3
+omegaconf==2.3.0
+PyYAML==6.0.3
+debugpy==1.8.20
+psutil==7.2.2
+ninja==1.13.0
+nvidia-cublas-cu12==12.4.5.8
+triton==3.2.0
+antlr4-python3-runtime==4.9.3
+peft==0.18.1
+aiosignal==1.4.0
+accelerate==1.12.0
+liger_kernel==0.7.0
+anyio==4.12.1
+decord==0.6.0
+h11==0.16.0
+urllib3==2.6.3
+wcwidth==0.6.0
+networkx==3.4.2
+huggingface_hub==0.36.2
+tzdata==2025.3
+xformers==0.0.29.post3
+tabulate==0.9.0
+certifi==2026.1.4
+tensorboard==2.20.0
+matplotlib-inline==0.2.1
+pip==26.0.1
+smmap==5.0.2
+nvidia-cuda-cupti-cu12==12.4.127
+Markdown==3.10.2
+nvidia-nccl-cu12==2.21.5
+aiohttp==3.13.3
+scikit-learn==1.7.2
+nvidia-cutlass-dsl==4.4.0
+setuptools==82.0.0
+jupyter_core==5.9.1
+asyncio==4.0.0
+einops==0.8.2
+pydantic==2.12.5
+av==16.1.0
+protobuf==6.33.5
+evaluate==0.4.6
+Jinja2==3.1.6
+exceptiongroup==1.3.1
+ipython==8.38.0
+sentry-sdk==2.53.0
+httpx==0.28.1
+msgpack==1.1.2
+platformdirs==4.9.2
+hjson==3.1.0
+torchvision==0.21.0
+parso==0.8.6
+cuda-bindings==13.1.1
+pexpect==4.9.0
+threadpoolctl==3.6.0
+pyarrow==23.0.1
+jedi==0.19.2
+nvidia-cuda-nvrtc-cu12==12.4.127
+Werkzeug==3.1.5
+fsspec==2025.10.0
+pydantic_core==2.41.5
+GitPython==3.1.46
+propcache==0.4.1
+mpmath==1.3.0
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-cusparse-cu12==12.3.1.170
+hf-xet==1.2.0
+charset-normalizer==3.4.4
+dill==0.4.0
+datasets==4.5.0
+py-cpuinfo==9.0.0
+sentencepiece==0.2.1
+latex2sympy2_extended==1.11.0
+jupyter_client==8.8.0
+pyzmq==27.1.0
+numpy==2.2.6
+gitdb==4.0.12
+grpcio==1.78.0
+packaging==26.0
+requests==2.32.5
+nvidia-cudnn-cu12==9.1.0.70
+httpcore==1.0.9
+traitlets==5.14.3
+filelock==3.24.2
+ptyprocess==0.7.0
+pillow==12.1.1
+torch==2.6.0
+more-itertools==10.8.0
+wheel==0.46.3
+jaraco.text==4.0.0
+zipp==3.23.0
+autocommand==2.2.2
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+tomli==2.4.0
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+platformdirs==4.4.0
+packaging==26.0

reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-metadata.json

@@ -0,0 +1,98 @@
+{
+  "os":  "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.19",
+  "startedAt":  "2026-05-29T18:23:15.496253Z",
+  "args":  [
+    "config=../configs/rl_lavida_video_reversal.yaml",
+    "wandb.run_id=fmmkx1a5",
+    "experiment.function=train",
+    "evaluation.block_size=None",
+    "evaluation.top_k=None",
+    "evaluation.remasking_strategy=None",
+    "experiment.current_epoch=6"
+  ],
+  "program":  "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py",
+  "codePath":  "reward/rl_reward_video_reversal.py",
+  "codePathLocal":  "rl_reward_video_reversal.py",
+  "git":  {
+    "remote":  "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
+    "commit":  "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
+  },
+  "email":  "shixi_runqing@staff.weibo.com",
+  "root":  "/data4/runqing/dLLM-RL/reward",
+  "host":  "b1301b38c75f",
+  "executable":  "/data8/runqing/miniconda/envs/dllm-rl/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H800",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "806578724864",
+      "used":  "625479712768"
+    }
+  },
+  "memory":  {
+    "total":  "2164113764352"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2775489f-18c8-2328-8829-c46238575379"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
+    }
+  ],
+  "cudaVersion":  "12.8",
+  "writerId":  "rit67u7afan4qam30286z3mhxid958av"
+}

reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_timestamp":1.7800760271916904e+09,"reward/train_step":1,"_step":26,"_wandb":{"runtime":4222},"train/learning_rate":7.985792958513929e-07,"reward/t_grpo_bonus":0.2718750000000003,"train/loss":-0.37430867552757263,"reward/avg_response_length":124.568359375,"reward/unbiased_bonus":0.09374999999999996,"reward/acc":0.396484375,"_runtime":4222}

reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug-core.log

@@ -0,0 +1,18 @@
+{"time":"2026-05-30T02:23:15.564585624+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpvrnq85bv/port-1972975.txt","pid":1972975,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-05-30T02:23:15.567024905+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1972975}
+{"time":"2026-05-30T02:23:15.56702786+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1972975-1973131-39973928/socket","Net":"unix"}}
+{"time":"2026-05-30T02:23:15.745166446+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-05-30T02:23:15.755860291+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T02:23:16.055380273+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T02:23:16.546440823+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fu9pap2v31pp"}
+{"time":"2026-05-30T02:23:16.730663414+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fu9pap2v31pp"}
+{"time":"2026-05-30T02:23:16.732391613+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T02:23:16.733226984+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T02:23:16.733240787+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-05-30T02:23:16.733255106+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-05-30T02:23:16.733267833+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-05-30T02:23:16.733276961+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-05-30T02:23:16.733350086+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1972975-1973131-39973928/socket","Net":"unix"}}
+{"time":"2026-05-30T02:23:16.733430555+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-05-30T02:23:16.733447157+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-05-30T02:23:16.733468785+08:00","level":"INFO","msg":"server is closed"}

reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug-internal.log

@@ -0,0 +1,28 @@
+{"time":"2026-05-30T02:23:15.756016562+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
+{"time":"2026-05-30T02:23:16.054970322+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
+{"time":"2026-05-30T02:23:16.055173873+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"}
+{"time":"2026-05-30T02:23:16.055246728+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T02:23:16.055363043+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"}
+{"time":"2026-05-30T02:23:16.055380438+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T02:23:16.055393379+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T02:23:16.059603255+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
+{"time":"2026-05-30T02:23:16.335493869+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-30T02:23:16.545805834+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":6,"current":27}
+{"time":"2026-05-30T02:23:16.677526751+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=821bfbf28a105fa324036c7aab937e5955c74cc8ce99454b4e05e18681bed874"}
+{"time":"2026-05-30T02:23:16.677717912+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml</Resource><RequestId>18B41C6A8D404808</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=821bfbf28a105fa324036c7aab937e5955c74cc8ce99454b4e05e18681bed874, Size: 7994}"}
+{"time":"2026-05-30T02:23:16.678145344+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9936bf6fb1a6034661afb45fccd887766fa50da78ed3dd0dd0a9c75ac6738e28"}
+{"time":"2026-05-30T02:23:16.67830001+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json</Resource><RequestId>18B41C6A8C90601F</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=9936bf6fb1a6034661afb45fccd887766fa50da78ed3dd0dd0a9c75ac6738e28, Size: 343}"}
+{"time":"2026-05-30T02:23:16.679660464+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d1a151f9161d8724a96f5cf5290917f0f27f95e56c278d8aeb0bef59702e3624"}
+{"time":"2026-05-30T02:23:16.679851441+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log</Resource><RequestId>18B41C6A8D3A116E</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=d1a151f9161d8724a96f5cf5290917f0f27f95e56c278d8aeb0bef59702e3624, Size: 110}"}
+{"time":"2026-05-30T02:23:16.68138543+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=63d9ebec3584b60becb67d7402bcda98014208276ed41ff4039e920fa99f4335"}
+{"time":"2026-05-30T02:23:16.681533107+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json</Resource><RequestId>18B41C6A8D37E2C7</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=63d9ebec3584b60becb67d7402bcda98014208276ed41ff4039e920fa99f4335, Size: 2849}"}
+{"time":"2026-05-30T02:23:16.685224569+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7f134f6231481934027ed53fe2c3b34d2356fabc7496ac5ed8bf95a505cf10e5"}
+{"time":"2026-05-30T02:23:16.685330732+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py</Resource><RequestId>18B41C6A8C9F1741</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=7f134f6231481934027ed53fe2c3b34d2356fabc7496ac5ed8bf95a505cf10e5, Size: 14790}"}
+{"time":"2026-05-30T02:23:16.686129396+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=33b21b6b3a8f6747127fcbaccf98aee64e2a8e82b3fbfe76e620d358ad90e198"}
+{"time":"2026-05-30T02:23:16.68618286+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt</Resource><RequestId>18B41C6A8CF03B7E</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T182316Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=33b21b6b3a8f6747127fcbaccf98aee64e2a8e82b3fbfe76e620d358ad90e198, Size: 2982}"}
+{"time":"2026-05-30T02:23:16.686494398+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-30T02:23:16.727909569+08:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-05-30T02:23:16.732429923+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"}
+{"time":"2026-05-30T02:23:16.732456013+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T02:23:16.732531196+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T02:23:16.732548218+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"}

reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug.log

@@ -0,0 +1,25 @@
+2026-05-30 02:23:15,499 INFO    MainThread:1972975 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
+2026-05-30 02:23:15,499 INFO    MainThread:1972975 [wandb_setup.py:_flush():81] Configure stats pid to 1972975
+2026-05-30 02:23:15,499 INFO    MainThread:1972975 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-30 02:23:15,499 INFO    MainThread:1972975 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug.log
+2026-05-30 02:23:15,499 INFO    MainThread:1972975 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_022315-fmmkx1a5/logs/debug-internal.log
+2026-05-30 02:23:15,499 INFO    MainThread:1972975 [wandb_init.py:init():844] calling init triggers
+2026-05-30 02:23:15,499 INFO    MainThread:1972975 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 6, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}}
+2026-05-30 02:23:15,499 INFO    MainThread:1972975 [wandb_init.py:init():892] starting backend
+2026-05-30 02:23:15,745 INFO    MainThread:1972975 [wandb_init.py:init():895] sending inform_init request
+2026-05-30 02:23:15,751 INFO    MainThread:1972975 [wandb_init.py:init():903] backend started and connected
+2026-05-30 02:23:15,753 INFO    MainThread:1972975 [wandb_init.py:init():973] updated telemetry
+2026-05-30 02:23:15,756 INFO    MainThread:1972975 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-30 02:23:16,328 INFO    MainThread:1972975 [wandb_init.py:init():1037] run resumed
+2026-05-30 02:23:16,333 INFO    MainThread:1972975 [wandb_init.py:init():1042] starting run threads in backend
+2026-05-30 02:23:16,425 INFO    MainThread:1972975 [wandb_run.py:_console_start():2524] atexit reg
+2026-05-30 02:23:16,425 INFO    MainThread:1972975 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-05-30 02:23:16,425 INFO    MainThread:1972975 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-05-30 02:23:16,425 INFO    MainThread:1972975 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-05-30 02:23:16,428 INFO    MainThread:1972975 [wandb_init.py:init():1082] run started, returning control to user process
+2026-05-30 02:23:16,545 INFO    MainThread:1972975 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5
+2026-05-30 02:23:16,545 INFO    MainThread:1972975 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
+2026-05-30 02:23:16,546 INFO    MainThread:1972975 [wandb_run.py:_restore():2472] restore
+2026-05-30 02:23:16,546 INFO    MainThread:1972975 [wandb_run.py:_restore():2478] restore done
+2026-05-30 02:23:16,731 INFO    MainThread:1972975 [wandb_run.py:_footer_sync_info():3868] logging synced files

reward/wandb/run-20260530_042845-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py

@@ -0,0 +1,339 @@
+import json
+import math_utils_v
+import nest_asyncio
+from concurrent.futures import ThreadPoolExecutor
+import asyncio
+import re
+from termcolor import cprint
+from omegaconf import MISSING
+from omegaconf import DictConfig, ListConfig, OmegaConf
+from typing import List, Union
+import os
+from collections import defaultdict
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, REPO_ROOT)
+
+import wandb
+from train.utils import flatten_omega_conf, ensure_wandb_run_id
+
+def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
+    p = sample.get(f"{media_type}_abs_path")
+    if p is None:
+        p = sample.get(media_type) or sample.get(f"{media_type}_path")
+        if p is None:
+            return None
+        if not os.path.isabs(p) and dataset_root:
+            p = os.path.join(dataset_root, p)
+    try:
+        return os.path.abspath(p)
+    except Exception:
+        return p
+
+def get_config():
+    cli_conf = OmegaConf.from_cli()
+    yaml_conf = OmegaConf.load(cli_conf.config)
+    conf = OmegaConf.merge(yaml_conf, cli_conf)
+    return conf
+
+def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]:
+    token = '<|endoftext|>'
+    if isinstance(obj, str):
+        return obj.replace(token, '')
+    if isinstance(obj, list):
+        return [s.replace(token, '') if isinstance(s, str) else s for s in obj]
+    raise TypeError("Expected str or list of str")
+
+
+if __name__ == "__main__":
+
+    config = get_config()
+    run = None
+    if config.experiment.function == "train":
+        existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None)
+        run_id = ensure_wandb_run_id(config)
+        resume_wandb_run = config.wandb.resume
+        if existing_run_id is None:
+            resume_wandb_run = False
+        wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
+        run = wandb.init(
+            project=config.experiment.project,
+            entity=config.wandb.get("entity", None),
+            id=run_id,
+            resume=resume_wandb_run,
+            config=wandb_config,
+            reinit=True,
+            name=config.experiment.project,
+        )
+
+    project_name = config.experiment.project
+    
+    if config.experiment.current_epoch == 1:
+        pretrained_model = config.model.pretrained_model
+    else:
+        pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name
+    
+    if config.experiment.function == "train":
+        shrink = config.training.shrink
+        dataset = config.dataset.train_dataset
+        outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}"
+        
+    elif config.experiment.function == "evaluation":
+        dataset = config.evaluation.eval_dataset
+        outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}"
+    
+    file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json"
+
+    with open(file_name, 'r') as f:
+        data = json.load(f)
+
+    index_list = []
+    extracted_output_list = []
+    response_length_list = []
+    for i in range(len(data)):
+        data[i]["correctness"] = []
+        response_length_list += data[i]["response_length"]
+        index_list += [i] * len(data[i]["extracted_output"])
+        extracted_output_list += remove_endoftext(data[i]["extracted_output"])
+
+    nest_asyncio.apply()
+    
+    async def get_correctness_all():
+        executor = ThreadPoolExecutor(max_workers=64)
+        tasks = []
+        
+        for p in range(len(index_list)):
+            pred = extracted_output_list[p]
+            idx = index_list[p]
+            
+            # 选择题正确性匹配逻辑
+            if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse":
+                gt = data[idx].get("response", "")
+                
+                async def mcq_check(p_str, g_str):
+                    if p_str in g_str or g_str in p_str:
+                        return True
+
+                    p_str = str(p_str).strip()
+                    g_str = str(g_str).strip()
+                    p_match = re.search(r'^([A-Z])[\.\s]', p_str)
+                    g_match = re.search(r'^([A-Z])[\.\s]', g_str)
+                    if p_match and g_match:
+                        return p_match.group(1) == g_match.group(1)
+                    return g_str in p_str
+                
+                tasks.append(mcq_check(pred, gt))
+            else:
+                gt = data[idx].get("ground_truth_answer", "")
+                tasks.append(math_utils_v.is_equal(pred, gt, executor))
+                
+        return await asyncio.gather(*tasks)
+
+    correctness_all = asyncio.run(get_correctness_all())
+    for p, ok in enumerate(correctness_all):
+        index_i = index_list[p]
+        data[index_i]["correctness"].append(bool(ok))
+
+    # =========================================================================
+    # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励
+    # =========================================================================
+    
+    # 1. 按照 data_id 分组映射索引
+    grouped_data = defaultdict(lambda: {
+        "forward_False": [], "backward_False": [], 
+        "forward_True": [],  "backward_True": []
+    })
+
+    for i in range(len(data)):
+        d_id = data[i].get("data_id")
+        if d_id is None:
+            continue
+        r_type = data[i].get("reversal_type", "forward")
+        is_shuf = str(data[i].get("is_shuffled", False))
+        grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i)
+
+    # 计算并应用群组奖励
+    all_temporal_bonuses = []  # 新增：记录所有的时序红利
+    all_unbiased_bonuses = []  # 新增：记录所有的无偏红利
+
+    # 2. 计算并应用群组奖励
+    for d_id, group in grouped_data.items():
+        
+        def get_acc(key):
+            if not group[key]: return 0.0
+            idx = group[key][0]
+            corr = data[idx]["correctness"]
+            return sum(corr) / len(corr) if len(corr) > 0 else 0.0
+
+        # 获取四组情况的正确率
+        acc_f_ord = get_acc("forward_False")
+        acc_f_shuf = get_acc("forward_True")
+        acc_b_ord = get_acc("backward_False")
+        acc_b_shuf = get_acc("backward_True")
+
+        # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3
+        f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0
+        b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0
+
+        # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3
+        unbiased_bonus = 0.0
+        if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1):
+            unbiased_bonus = 0.3
+
+        # 新增：将当前组的奖励记录下来用于统计
+        if group["forward_False"]:
+            all_temporal_bonuses.append(f_temporal_bonus)
+            all_unbiased_bonuses.append(unbiased_bonus)
+        if group["backward_False"]:
+            all_temporal_bonuses.append(b_temporal_bonus)
+            all_unbiased_bonuses.append(unbiased_bonus)
+
+        # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本
+        def apply_bonus(key, temporal_bonus, unbiased_bonus):
+            if not group[key]: return
+            idx = group[key][0]
+            raw_rewards = []
+            for is_correct in data[idx]["correctness"]:
+                if is_correct:
+                    # 基础分 1.0 + 时序红利 + 无偏红利
+                    raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus)
+                else:
+                    raw_rewards.append(0.0)
+            data[idx]["raw_rewards"] = raw_rewards
+
+        apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus)
+        apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus)
+
+    # =========================================================================
+
+    def z_score_normalize(lst):
+        mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0
+        std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0
+        if std == 0:
+            return [0 for _ in lst]
+        return [(x - mean) / std for x in lst]
+
+    final_data = []
+    ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据
+    
+    for i in range(len(data)):
+        # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习
+        if data[i].get("is_shuffled", False):
+            continue
+            
+        ordered_correctness.extend(data[i]["correctness"])
+
+        # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数
+        raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]])
+        lengths = data[i]["response_length"]
+        
+        # 长度惩罚
+        if config.reward.strict_len_check:
+            for j in range(len(lengths)):
+                if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5:
+                    raw_rewards[j] = 0.0
+                if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5:
+                    raw_rewards[j] = 0.0
+
+        # Z-Score 优势归一化
+        rewards = z_score_normalize(raw_rewards)
+        data[i]["rewards"] = rewards
+
+        if config.experiment.function == "train":
+            proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0
+            # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本
+            if proportion > 0.95 or proportion < 0.05:
+                continue
+
+            for j in range(len(rewards)):
+                data_i = {}
+                data_i["prompt"] = data[i]["prompt"]
+                data_i["reward"] = rewards[j]
+                data_i["response"] = data[i]["full_output"][j]
+                data_i["step_map"] = data[i]["step_map"][j]
+                
+                # 保留其他需要的上下文 ID
+                data_i["data_id"] = data[i].get("data_id")
+                data_i["reversal_type"] = data[i].get("reversal_type")
+                
+                if "image_token_ids" in data[i]:
+                    data_i["image_token_ids"] = data[i]["image_token_ids"]
+                
+                p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video")
+                if p_video_abs is not None:
+                    data_i["video_abs_path"] = p_video_abs
+                
+                p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image")
+                if p_image_abs is not None:
+                    data_i["image_abs_path"] = p_image_abs
+
+                final_data.append(data_i)
+
+        if config.experiment.function == "evaluation":
+            data[i]["step_map"] = []
+
+    if config.experiment.function == "train":
+        with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f:
+            json.dump(final_data, f, indent=2, ensure_ascii=False)
+
+
+    os.makedirs(os.path.dirname(file_name), exist_ok=True)
+    with open(file_name, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+
+
+    outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt"
+    os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True)
+    with open(outputs_result_name, "a") as f:
+        # Save + print
+        def save_and_print(text):
+            cprint("\n\n\n" + text, color="green")
+            f.write(text + "\n")
+        
+        # 只计算有序组（有效学习组）的准确率来作为监控指标
+        acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan")
+        avg_len = sum(response_length_list)/len(response_length_list)
+
+        # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值
+        avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0
+        avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0
+
+        output_text = f"train step: {config.experiment.current_epoch}  "
+        if config.experiment.function == "train":
+            if config.model.model_base != "sdar" and config.model.model_base != "trado":
+                output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy}  block_size: {config.rollout.block_size}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+            else:
+                output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy}  top_k: {config.rollout.top_k}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+        else:
+            if config.model.model_base != "sdar" and config.model.model_base != "trado":
+                output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy}  block_size: {config.evaluation.block_size}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+            else:
+                output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy}  top_k: {config.evaluation.top_k}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+        save_and_print(output_text)
+
+    if config.experiment.function == "train":
+        csv_file_path = f"../{project_name}/results/all_rewards.csv"
+        
+        # 如果是第1个 epoch/step，或者是新文件，就先写入表头
+        write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1
+        
+        with open(csv_file_path, "a", encoding="utf-8") as f_csv:
+            if write_header:
+                f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n")
+            
+            # 按行追加当前 step 的数据
+            f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n")
+
+        if run is not None:
+            wandb.log(
+                {
+                    "reward/acc": acc,
+                    "reward/t_grpo_bonus": avg_temp,
+                    "reward/unbiased_bonus": avg_unbias,
+                    "reward/avg_response_length": avg_len,
+                    "reward/train_step": config.experiment.current_epoch,
+                },
+                step=config.experiment.current_epoch,
+            )
+            wandb.finish()

reward/wandb/run-20260530_042845-fmmkx1a5/files/config.yaml

@@ -0,0 +1,281 @@
+_wandb:
+    value:
+        cli_version: 0.25.0
+        code_path: code/reward/rl_reward_video_reversal.py
+        e:
+            iv8zvsdp3bk0inm3f71d72crajbgfbvu:
+                args:
+                    - config=../configs/rl_lavida_video_reversal.yaml
+                    - wandb.run_id=fmmkx1a5
+                    - experiment.function=train
+                    - evaluation.block_size=None
+                    - evaluation.top_k=None
+                    - evaluation.remasking_strategy=None
+                    - experiment.current_epoch=8
+                codePath: reward/rl_reward_video_reversal.py
+                codePathLocal: rl_reward_video_reversal.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "806578724864"
+                        used: "625480916992"
+                email: shixi_runqing@staff.weibo.com
+                executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python
+                git:
+                    commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255
+                    remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git
+                gpu: NVIDIA H800
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-2775489f-18c8-2328-8829-c46238575379
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd
+                host: b1301b38c75f
+                memory:
+                    total: "2164113764352"
+                os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35
+                program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py
+                python: CPython 3.10.19
+                root: /data4/runqing/dLLM-RL/reward
+                startedAt: "2026-05-29T20:28:45.290476Z"
+                writerId: iv8zvsdp3bk0inm3f71d72crajbgfbvu
+        m: []
+        python_version: 3.10.19
+        t:
+            "1":
+                - 1
+                - 41
+            "2":
+                - 1
+                - 41
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 16
+                - 61
+                - 62
+            "4": 3.10.19
+            "5": 0.25.0
+            "10":
+                - 20
+            "12": 0.25.0
+            "13": linux-x86_64
+config:
+    value: ../configs/rl_lavida_video_reversal.yaml
+dataset.data_type:
+    value: video_reverse
+dataset.optimization_data:
+    value: rl_data
+dataset.train_dataset:
+    value: video_reversal_rl_train
+evaluation.batch_size:
+    value: 1
+evaluation.block_size:
+    value: None
+evaluation.data_type:
+    value: video_reverse
+evaluation.dynamic_threshold:
+    value: 0.9
+evaluation.eval_dataset:
+    value: video_reversal_rl_eval
+evaluation.further_horizon:
+    value: 128
+evaluation.if_eval:
+    value: false
+evaluation.max_gen_length:
+    value: 128
+evaluation.num_response_per_task:
+    value: 3
+evaluation.remasking_strategy:
+    value: None
+evaluation.remasking_strategy.0:
+    value: low_confidence_static
+evaluation.remasking_strategy.1:
+    value: low_confidence_dynamic
+evaluation.steps:
+    value: 128
+evaluation.target:
+    value: confidence
+evaluation.temperature:
+    value: 0.1
+evaluation.top_k:
+    value: None
+evaluation.use_cache:
+    value: true
+execute.num_chunk:
+    value: 128
+experiment.current_epoch:
+    value: 8
+experiment.deepspeed_file:
+    value: 1_node_8_gpus_deepspeed_zero2
+experiment.eval_every:
+    value: 5
+experiment.function:
+    value: train
+experiment.logging_dir:
+    value: rl_lavida_video_reversal/logs
+experiment.node_index:
+    value: 0
+experiment.num_node:
+    value: 1
+experiment.project:
+    value: rl_lavida_video_reversal
+experiment.save_every:
+    value: 10
+experiment.start_from_scratch:
+    value: true
+experiment.total_step:
+    value: 30
+lr_scheduler.params.learning_rate:
+    value: 1e-06
+lr_scheduler.params.min_lr_scale:
+    value: 1
+lr_scheduler.params.warmup_steps:
+    value: 0
+lr_scheduler.scheduler:
+    value: cosine
+model.image_resolution:
+    value: 512
+model.model_base:
+    value: lavida
+model.optimized_name:
+    value: optimized
+model.pretrained_model:
+    value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized
+model.vq_model_path:
+    value: /data4/runqing/hf/magvitv2
+optimizer.name:
+    value: adamw
+optimizer.params.beta1:
+    value: 0.9
+optimizer.params.beta2:
+    value: 0.999
+optimizer.params.epsilon:
+    value: 1e-08
+optimizer.params.learning_rate:
+    value: 1e-06
+optimizer.params.scale_lr:
+    value: false
+optimizer.params.weight_decay:
+    value: 0
+reward.answer_must_in_box:
+    value: true
+reward.strict_len_check:
+    value: false
+rollout.batch_size:
+    value: 1
+rollout.block_size:
+    value: 32
+rollout.dynamic_threshold:
+    value: 0.9
+rollout.further_horizon:
+    value: 128
+rollout.max_gen_length:
+    value: 128
+rollout.num_response_per_task:
+    value: 8
+rollout.num_task_per_step:
+    value: 128
+rollout.remasking_strategy:
+    value: low_confidence_dynamic
+rollout.steps:
+    value: 128
+rollout.target:
+    value: confidence
+rollout.temperature:
+    value: 0.8
+rollout.use_cache:
+    value: true
+system.HF_HOME:
+    value: null
+system.HTTP_PROXY:
+    value: null
+system.base_dir:
+    value: /data4/runqing/dLLM-RL
+system.env_name:
+    value: /data8/runqing/miniconda/envs/dllm-rl
+system.envs_dir:
+    value: /data8/runqing/miniconda/envs/dllm-rl
+training.batch_size_lm:
+    value: 1
+training.beta:
+    value: 0.01
+training.enable_tf32:
+    value: true
+training.eps:
+    value: 0.2
+training.gradient_accumulation_steps:
+    value: 2
+training.lower_p:
+    value: 0.35
+training.mask_times_per_sample:
+    value: 2
+training.max_gen_length:
+    value: 128
+training.max_grad_norm:
+    value: 1
+training.max_prompt_len:
+    value: 512
+training.method:
+    value: random_masking
+training.mixed_precision:
+    value: bf16
+training.num_train_epochs:
+    value: 1
+training.post_num:
+    value: 1
+training.seed:
+    value: 10086
+training.shrink:
+    value: 8
+training.upper_p:
+    value: 0.9
+training.use_kl_estimator_k3:
+    value: true
+wandb.entity:
+    value: null
+wandb.resume:
+    value: auto
+wandb.run_id:
+    value: fmmkx1a5

reward/wandb/run-20260530_042845-fmmkx1a5/files/output.log

@@ -0,0 +1,4 @@
+
+
+
+train step: 8  remasking_strategy: low_confidence_dynamic  block_size: 32  acc: 0.4688  avg length: 126.02

reward/wandb/run-20260530_042845-fmmkx1a5/files/requirements.txt

@@ -0,0 +1,157 @@
+transformers==4.52.4
+nvidia-nvtx-cu12==12.4.127
+deepspeed==0.18.6
+typing-inspection==0.4.2
+scipy==1.15.3
+MarkupSafe==3.0.3
+nvidia-cuda-runtime-cu12==12.4.127
+detoxify==0.5.2
+yarl==1.22.0
+tornado==6.5.4
+nvidia-curand-cu12==10.3.5.147
+nest-asyncio==1.6.0
+safetensors==0.7.0
+async-timeout==5.0.1
+typing_extensions==4.15.0
+asttokens==3.0.1
+cuda-python==13.1.1
+executing==2.2.1
+nvidia-cudnn-frontend==1.18.0
+comm==0.2.3
+tokenizers==0.21.4
+six==1.17.0
+Pygments==2.19.2
+nvidia-cusparselt-cu12==0.6.2
+apache-tvm-ffi==0.1.8.post2
+flash_attn==2.7.4.post1
+tqdm==4.67.3
+tensorboard-data-server==0.7.2
+annotated-types==0.7.0
+math-verify==0.9.0
+idna==3.11
+regex==2026.1.15
+nvidia-cutlass-dsl-libs-base==4.4.0
+flashinfer-python==0.6.3
+click==8.3.1
+pytz==2025.2
+wheel==0.46.3
+ipykernel==7.2.0
+stack-data==0.6.3
+absl-py==2.4.0
+wandb==0.25.0
+pandas==2.3.3
+termcolor==3.3.0
+sympy==1.13.1
+joblib==1.5.3
+prompt_toolkit==3.0.52
+attrs==25.4.0
+nvidia-cusolver-cu12==11.6.1.9
+xxhash==3.6.0
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+frozenlist==1.8.0
+nvidia-ml-py==13.590.48
+multiprocess==0.70.18
+decorator==5.2.1
+cuda-pathfinder==1.3.4
+multidict==6.7.1
+pure_eval==0.2.3
+nvidia-cufft-cu12==11.2.1.3
+omegaconf==2.3.0
+PyYAML==6.0.3
+debugpy==1.8.20
+psutil==7.2.2
+ninja==1.13.0
+nvidia-cublas-cu12==12.4.5.8
+triton==3.2.0
+antlr4-python3-runtime==4.9.3
+peft==0.18.1
+aiosignal==1.4.0
+accelerate==1.12.0
+liger_kernel==0.7.0
+anyio==4.12.1
+decord==0.6.0
+h11==0.16.0
+urllib3==2.6.3
+wcwidth==0.6.0
+networkx==3.4.2
+huggingface_hub==0.36.2
+tzdata==2025.3
+xformers==0.0.29.post3
+tabulate==0.9.0
+certifi==2026.1.4
+tensorboard==2.20.0
+matplotlib-inline==0.2.1
+pip==26.0.1
+smmap==5.0.2
+nvidia-cuda-cupti-cu12==12.4.127
+Markdown==3.10.2
+nvidia-nccl-cu12==2.21.5
+aiohttp==3.13.3
+scikit-learn==1.7.2
+nvidia-cutlass-dsl==4.4.0
+setuptools==82.0.0
+jupyter_core==5.9.1
+asyncio==4.0.0
+einops==0.8.2
+pydantic==2.12.5
+av==16.1.0
+protobuf==6.33.5
+evaluate==0.4.6
+Jinja2==3.1.6
+exceptiongroup==1.3.1
+ipython==8.38.0
+sentry-sdk==2.53.0
+httpx==0.28.1
+msgpack==1.1.2
+platformdirs==4.9.2
+hjson==3.1.0
+torchvision==0.21.0
+parso==0.8.6
+cuda-bindings==13.1.1
+pexpect==4.9.0
+threadpoolctl==3.6.0
+pyarrow==23.0.1
+jedi==0.19.2
+nvidia-cuda-nvrtc-cu12==12.4.127
+Werkzeug==3.1.5
+fsspec==2025.10.0
+pydantic_core==2.41.5
+GitPython==3.1.46
+propcache==0.4.1
+mpmath==1.3.0
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-cusparse-cu12==12.3.1.170
+hf-xet==1.2.0
+charset-normalizer==3.4.4
+dill==0.4.0
+datasets==4.5.0
+py-cpuinfo==9.0.0
+sentencepiece==0.2.1
+latex2sympy2_extended==1.11.0
+jupyter_client==8.8.0
+pyzmq==27.1.0
+numpy==2.2.6
+gitdb==4.0.12
+grpcio==1.78.0
+packaging==26.0
+requests==2.32.5
+nvidia-cudnn-cu12==9.1.0.70
+httpcore==1.0.9
+traitlets==5.14.3
+filelock==3.24.2
+ptyprocess==0.7.0
+pillow==12.1.1
+torch==2.6.0
+more-itertools==10.8.0
+wheel==0.46.3
+jaraco.text==4.0.0
+zipp==3.23.0
+autocommand==2.2.2
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+tomli==2.4.0
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+platformdirs==4.4.0
+packaging==26.0

reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-metadata.json

@@ -0,0 +1,98 @@
+{
+  "os":  "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.19",
+  "startedAt":  "2026-05-29T20:28:45.290476Z",
+  "args":  [
+    "config=../configs/rl_lavida_video_reversal.yaml",
+    "wandb.run_id=fmmkx1a5",
+    "experiment.function=train",
+    "evaluation.block_size=None",
+    "evaluation.top_k=None",
+    "evaluation.remasking_strategy=None",
+    "experiment.current_epoch=8"
+  ],
+  "program":  "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py",
+  "codePath":  "reward/rl_reward_video_reversal.py",
+  "codePathLocal":  "rl_reward_video_reversal.py",
+  "git":  {
+    "remote":  "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
+    "commit":  "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
+  },
+  "email":  "shixi_runqing@staff.weibo.com",
+  "root":  "/data4/runqing/dLLM-RL/reward",
+  "host":  "b1301b38c75f",
+  "executable":  "/data8/runqing/miniconda/envs/dllm-rl/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H800",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "806578724864",
+      "used":  "625480916992"
+    }
+  },
+  "memory":  {
+    "total":  "2164113764352"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2775489f-18c8-2328-8829-c46238575379"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
+    }
+  ],
+  "cudaVersion":  "12.8",
+  "writerId":  "iv8zvsdp3bk0inm3f71d72crajbgfbvu"
+}

reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-summary.json

@@ -0,0 +1 @@
+{"reward/acc":0.396484375,"_runtime":5992,"reward/avg_response_length":124.568359375,"train/loss":-0.37430867552757263,"reward/train_step":1,"_timestamp":1.7800760271916904e+09,"_wandb":{"runtime":5992},"reward/t_grpo_bonus":0.2718750000000003,"train/learning_rate":7.985792958513929e-07,"reward/unbiased_bonus":0.09374999999999996,"_step":26}

reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug-core.log

@@ -0,0 +1,18 @@
+{"time":"2026-05-30T04:28:45.357187774+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpe2ektbof/port-2189265.txt","pid":2189265,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-05-30T04:28:45.357749747+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2189265}
+{"time":"2026-05-30T04:28:45.357737063+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2189265-2189429-3673607564/socket","Net":"unix"}}
+{"time":"2026-05-30T04:28:45.538534603+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-05-30T04:28:45.547157906+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T04:28:45.852526644+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T04:28:46.370665657+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"viw1ypxpd56s"}
+{"time":"2026-05-30T04:28:46.550283848+08:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"viw1ypxpd56s"}
+{"time":"2026-05-30T04:28:46.551572545+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T04:28:46.552278813+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fmmkx1a5","id":"1(@)"}
+{"time":"2026-05-30T04:28:46.552287358+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-05-30T04:28:46.552297327+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-05-30T04:28:46.552305555+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-05-30T04:28:46.552320535+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-05-30T04:28:46.552335675+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-05-30T04:28:46.552423726+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-05-30T04:28:46.552482901+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2189265-2189429-3673607564/socket","Net":"unix"}}
+{"time":"2026-05-30T04:28:46.552538115+08:00","level":"INFO","msg":"server is closed"}

reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug-internal.log

@@ -0,0 +1,28 @@
+{"time":"2026-05-30T04:28:45.547363162+08:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
+{"time":"2026-05-30T04:28:45.851985924+08:00","level":"ERROR","msg":"featurechecker: failed to load features, all will be disabled","error":"input:4:3: Cannot query field \"features\" on type \"ServerInfo\".\n"}
+{"time":"2026-05-30T04:28:45.852256915+08:00","level":"INFO","msg":"stream: created new stream","id":"fmmkx1a5"}
+{"time":"2026-05-30T04:28:45.852386761+08:00","level":"INFO","msg":"handler: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T04:28:45.852507919+08:00","level":"INFO","msg":"stream: started","id":"fmmkx1a5"}
+{"time":"2026-05-30T04:28:45.852536962+08:00","level":"INFO","msg":"writer: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T04:28:45.852534157+08:00","level":"INFO","msg":"sender: started","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T04:28:45.858252069+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
+{"time":"2026-05-30T04:28:46.158666066+08:00","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-30T04:28:46.369997093+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":8,"current":27}
+{"time":"2026-05-30T04:28:46.501935127+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=dadf76cf88cfe1ec58e2105e56b323d6ac39772d8e5bb0ad7ed85e429bd4e85e"}
+{"time":"2026-05-30T04:28:46.502060021+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log</Resource><RequestId>18B42343B965E108</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/output.log, Name: output.log, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/output.log?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=dadf76cf88cfe1ec58e2105e56b323d6ac39772d8e5bb0ad7ed85e429bd4e85e, Size: 110}"}
+{"time":"2026-05-30T04:28:46.502198522+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6aa16e413100df81d7b2c203db0e968d616ffff4e373204c41041478c7fbfe13"}
+{"time":"2026-05-30T04:28:46.502298169+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e05b7e592252a0fe524561bedbb211f5123fe01e58341c518a6f90f8f8ccad28"}
+{"time":"2026-05-30T04:28:46.50236231+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml</Resource><RequestId>18B42343B966889E</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/config.yaml, Name: config.yaml, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/config.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=6aa16e413100df81d7b2c203db0e968d616ffff4e373204c41041478c7fbfe13, Size: 7994}"}
+{"time":"2026-05-30T04:28:46.502536921+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json</Resource><RequestId>18B42343B9A1978F</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-metadata.json, Name: wandb-metadata.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-metadata.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=e05b7e592252a0fe524561bedbb211f5123fe01e58341c518a6f90f8f8ccad28, Size: 2849}"}
+{"time":"2026-05-30T04:28:46.5035542+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=38aaaf3093af2965c378a7cf0d77ea697514c7df7c85d2d318d1ae08d7932f11"}
+{"time":"2026-05-30T04:28:46.503761198+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json</Resource><RequestId>18B42343B8DC7687</RequestId><HostId>29678716-7d96-4915-b26c-b7113941467d</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/wandb-summary.json, Name: wandb-summary.json, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/wandb-summary.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=38aaaf3093af2965c378a7cf0d77ea697514c7df7c85d2d318d1ae08d7932f11, Size: 343}"}
+{"time":"2026-05-30T04:28:46.505924826+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=326cb6c5ef32f7ec9d65c1165435635a9e1d486c435f164c34ec192c8dcdbd4d"}
+{"time":"2026-05-30T04:28:46.506020205+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt</Resource><RequestId>18B42343B9BD0B61</RequestId><HostId>4b5cbd82-31a6-4e32-8e3a-3b4f1cd8a94b</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/requirements.txt, Name: requirements.txt, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/requirements.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=326cb6c5ef32f7ec9d65c1165435635a9e1d486c435f164c34ec192c8dcdbd4d, Size: 2982}"}
+{"time":"2026-05-30T04:28:46.506408954+08:00","level":"ERROR","msg":"api: HTTP error","status":403,"method":"PUT","url":"http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=fa24244b919ee0c3b1debd877157ca47f4dfaf89a9c2928302cae995ae265dc6"}
+{"time":"2026-05-30T04:28:46.506468235+08:00","level":"ERROR","msg":"file transfer: upload: failed to upload: status: 403 Forbidden: body: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><Key>shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py</Key><BucketName>netdisk</BucketName><Resource>/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py</Resource><RequestId>18B42343B9587729</RequestId><HostId>17e81bf8-654d-4ce6-b556-0f7349513ee1</HostId></Error>","task":"DefaultUploadTask{FileKind: 1, Path: /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py, Name: code/reward/rl_reward_video_reversal.py, Url: http://jfs-s3-gateway.wml.weibo.com:80/netdisk/shixi_runqing/rl_lavida_video_reversal/fmmkx1a5/code/reward/rl_reward_video_reversal.py?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260529%2Fregion%2Fs3%2Faws4_request&X-Amz-Date=20260529T202846Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-User=shixi_runqing&X-Amz-Signature=fa24244b919ee0c3b1debd877157ca47f4dfaf89a9c2928302cae995ae265dc6, Size: 14790}"}
+{"time":"2026-05-30T04:28:46.506804552+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-30T04:28:46.547050766+08:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-05-30T04:28:46.551595229+08:00","level":"INFO","msg":"stream: closing","id":"fmmkx1a5"}
+{"time":"2026-05-30T04:28:46.551609585+08:00","level":"INFO","msg":"handler: closed","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T04:28:46.551670861+08:00","level":"INFO","msg":"sender: closed","stream_id":"fmmkx1a5"}
+{"time":"2026-05-30T04:28:46.551687822+08:00","level":"INFO","msg":"stream: closed","id":"fmmkx1a5"}

reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug.log

@@ -0,0 +1,25 @@
+2026-05-30 04:28:45,293 INFO    MainThread:2189265 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0
+2026-05-30 04:28:45,293 INFO    MainThread:2189265 [wandb_setup.py:_flush():81] Configure stats pid to 2189265
+2026-05-30 04:28:45,293 INFO    MainThread:2189265 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-30 04:28:45,293 INFO    MainThread:2189265 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug.log
+2026-05-30 04:28:45,293 INFO    MainThread:2189265 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data4/runqing/dLLM-RL/reward/wandb/run-20260530_042845-fmmkx1a5/logs/debug-internal.log
+2026-05-30 04:28:45,293 INFO    MainThread:2189265 [wandb_init.py:init():844] calling init triggers
+2026-05-30 04:28:45,293 INFO    MainThread:2189265 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'wandb.entity': None, 'wandb.resume': 'auto', 'wandb.run_id': 'fmmkx1a5', 'system.HTTP_PROXY': None, 'system.HF_HOME': None, 'system.env_name': '/data8/runqing/miniconda/envs/dllm-rl', 'system.envs_dir': '/data8/runqing/miniconda/envs/dllm-rl', 'system.base_dir': '/data4/runqing/dLLM-RL', 'experiment.project': 'rl_lavida_video_reversal', 'experiment.function': 'train', 'experiment.start_from_scratch': True, 'experiment.total_step': 30, 'experiment.save_every': 10, 'experiment.eval_every': 5, 'experiment.current_epoch': 8, 'experiment.deepspeed_file': '1_node_8_gpus_deepspeed_zero2', 'experiment.num_node': 1, 'experiment.node_index': 0, 'model.pretrained_model': '/data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized', 'model.optimized_name': 'optimized', 'model.model_base': 'lavida', 'model.vq_model_path': '/data4/runqing/hf/magvitv2', 'model.image_resolution': 512, 'dataset.train_dataset': 'video_reversal_rl_train', 'dataset.data_type': 'video_reverse', 'dataset.optimization_data': 'rl_data', 'rollout.num_task_per_step': 128, 'rollout.num_response_per_task': 8, 'rollout.temperature': 0.8, 'rollout.steps': 128, 'rollout.max_gen_length': 128, 'rollout.batch_size': 1, 'rollout.remasking_strategy': 'low_confidence_dynamic', 'rollout.target': 'confidence', 'rollout.dynamic_threshold': 0.9, 'rollout.block_size': 32, 'rollout.further_horizon': 128, 'rollout.use_cache': True, 'reward.answer_must_in_box': True, 'reward.strict_len_check': False, 'execute.num_chunk': 128, 'training.gradient_accumulation_steps': 2, 'training.batch_size_lm': 1, 'training.mixed_precision': 'bf16', 'training.enable_tf32': True, 'training.seed': 10086, 'training.num_train_epochs': 1, 'training.max_grad_norm': 1.0, 'training.method': 'random_masking', 'training.lower_p': 0.35, 'training.upper_p': 0.9, 'training.shrink': 8, 'training.post_num': 1, 'training.mask_times_per_sample': 2, 'training.max_gen_length': 128, 'training.max_prompt_len': 512, 'training.eps': 0.2, 'training.beta': 0.01, 'training.use_kl_estimator_k3': True, 'optimizer.name': 'adamw', 'optimizer.params.learning_rate': 1e-06, 'optimizer.params.scale_lr': False, 'optimizer.params.beta1': 0.9, 'optimizer.params.beta2': 0.999, 'optimizer.params.weight_decay': 0.0, 'optimizer.params.epsilon': 1e-08, 'lr_scheduler.scheduler': 'cosine', 'lr_scheduler.params.learning_rate': 1e-06, 'lr_scheduler.params.warmup_steps': 0, 'lr_scheduler.params.min_lr_scale': 1.0, 'evaluation.if_eval': False, 'evaluation.eval_dataset': 'video_reversal_rl_eval', 'evaluation.data_type': 'video_reverse', 'evaluation.num_response_per_task': 3, 'evaluation.temperature': 0.1, 'evaluation.steps': 128, 'evaluation.max_gen_length': 128, 'evaluation.batch_size': 1, 'evaluation.remasking_strategy': 'None', 'evaluation.target': 'confidence', 'evaluation.dynamic_threshold': 0.9, 'evaluation.block_size': 'None', 'evaluation.further_horizon': 128, 'evaluation.use_cache': True, 'evaluation.top_k': 'None', 'config': '../configs/rl_lavida_video_reversal.yaml', '_wandb': {'code_path': 'code/reward/rl_reward_video_reversal.py'}}
+2026-05-30 04:28:45,293 INFO    MainThread:2189265 [wandb_init.py:init():892] starting backend
+2026-05-30 04:28:45,538 INFO    MainThread:2189265 [wandb_init.py:init():895] sending inform_init request
+2026-05-30 04:28:45,543 INFO    MainThread:2189265 [wandb_init.py:init():903] backend started and connected
+2026-05-30 04:28:45,545 INFO    MainThread:2189265 [wandb_init.py:init():973] updated telemetry
+2026-05-30 04:28:45,548 INFO    MainThread:2189265 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-30 04:28:46,152 INFO    MainThread:2189265 [wandb_init.py:init():1037] run resumed
+2026-05-30 04:28:46,157 INFO    MainThread:2189265 [wandb_init.py:init():1042] starting run threads in backend
+2026-05-30 04:28:46,246 INFO    MainThread:2189265 [wandb_run.py:_console_start():2524] atexit reg
+2026-05-30 04:28:46,246 INFO    MainThread:2189265 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-05-30 04:28:46,246 INFO    MainThread:2189265 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-05-30 04:28:46,246 INFO    MainThread:2189265 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-05-30 04:28:46,250 INFO    MainThread:2189265 [wandb_init.py:init():1082] run started, returning control to user process
+2026-05-30 04:28:46,369 INFO    MainThread:2189265 [wandb_run.py:_finish():2291] finishing run shixi_runqing/rl_lavida_video_reversal/fmmkx1a5
+2026-05-30 04:28:46,370 INFO    MainThread:2189265 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
+2026-05-30 04:28:46,370 INFO    MainThread:2189265 [wandb_run.py:_restore():2472] restore
+2026-05-30 04:28:46,370 INFO    MainThread:2189265 [wandb_run.py:_restore():2478] restore done
+2026-05-30 04:28:46,550 INFO    MainThread:2189265 [wandb_run.py:_footer_sync_info():3868] logging synced files

reward/wandb/run-20260530_063905-fmmkx1a5/files/code/reward/rl_reward_video_reversal.py

@@ -0,0 +1,339 @@
+import json
+import math_utils_v
+import nest_asyncio
+from concurrent.futures import ThreadPoolExecutor
+import asyncio
+import re
+from termcolor import cprint
+from omegaconf import MISSING
+from omegaconf import DictConfig, ListConfig, OmegaConf
+from typing import List, Union
+import os
+from collections import defaultdict
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, REPO_ROOT)
+
+import wandb
+from train.utils import flatten_omega_conf, ensure_wandb_run_id
+
+def resolve_media_abs_path(sample: dict, dataset_root: str | None, media_type: str = "image") -> str | None:
+    p = sample.get(f"{media_type}_abs_path")
+    if p is None:
+        p = sample.get(media_type) or sample.get(f"{media_type}_path")
+        if p is None:
+            return None
+        if not os.path.isabs(p) and dataset_root:
+            p = os.path.join(dataset_root, p)
+    try:
+        return os.path.abspath(p)
+    except Exception:
+        return p
+
+def get_config():
+    cli_conf = OmegaConf.from_cli()
+    yaml_conf = OmegaConf.load(cli_conf.config)
+    conf = OmegaConf.merge(yaml_conf, cli_conf)
+    return conf
+
+def remove_endoftext(obj: Union[str, List[str]]) -> Union[str, List[str]]:
+    token = '<|endoftext|>'
+    if isinstance(obj, str):
+        return obj.replace(token, '')
+    if isinstance(obj, list):
+        return [s.replace(token, '') if isinstance(s, str) else s for s in obj]
+    raise TypeError("Expected str or list of str")
+
+
+if __name__ == "__main__":
+
+    config = get_config()
+    run = None
+    if config.experiment.function == "train":
+        existing_run_id = OmegaConf.select(config, "wandb.run_id", default=None)
+        run_id = ensure_wandb_run_id(config)
+        resume_wandb_run = config.wandb.resume
+        if existing_run_id is None:
+            resume_wandb_run = False
+        wandb_config = {k: v for k, v in flatten_omega_conf(config, resolve=True)}
+        run = wandb.init(
+            project=config.experiment.project,
+            entity=config.wandb.get("entity", None),
+            id=run_id,
+            resume=resume_wandb_run,
+            config=wandb_config,
+            reinit=True,
+            name=config.experiment.project,
+        )
+
+    project_name = config.experiment.project
+    
+    if config.experiment.current_epoch == 1:
+        pretrained_model = config.model.pretrained_model
+    else:
+        pretrained_model = "../" + project_name + "/ckpt/" + config.model.optimized_name
+    
+    if config.experiment.function == "train":
+        shrink = config.training.shrink
+        dataset = config.dataset.train_dataset
+        outputs_name = "rl-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}"
+        
+    elif config.experiment.function == "evaluation":
+        dataset = config.evaluation.eval_dataset
+        outputs_name = "eval-" + pretrained_model.replace("/", ".") + "-" + dataset + f"-step{config.experiment.current_epoch}"
+    
+    file_name = "../" + project_name + "/temp_data/outputs-" + outputs_name + ".json"
+
+    with open(file_name, 'r') as f:
+        data = json.load(f)
+
+    index_list = []
+    extracted_output_list = []
+    response_length_list = []
+    for i in range(len(data)):
+        data[i]["correctness"] = []
+        response_length_list += data[i]["response_length"]
+        index_list += [i] * len(data[i]["extracted_output"])
+        extracted_output_list += remove_endoftext(data[i]["extracted_output"])
+
+    nest_asyncio.apply()
+    
+    async def get_correctness_all():
+        executor = ThreadPoolExecutor(max_workers=64)
+        tasks = []
+        
+        for p in range(len(index_list)):
+            pred = extracted_output_list[p]
+            idx = index_list[p]
+            
+            # 选择题正确性匹配逻辑
+            if config.dataset.data_type == "video_reverse" or config.evaluation.data_type == "video_reverse":
+                gt = data[idx].get("response", "")
+                
+                async def mcq_check(p_str, g_str):
+                    if p_str in g_str or g_str in p_str:
+                        return True
+
+                    p_str = str(p_str).strip()
+                    g_str = str(g_str).strip()
+                    p_match = re.search(r'^([A-Z])[\.\s]', p_str)
+                    g_match = re.search(r'^([A-Z])[\.\s]', g_str)
+                    if p_match and g_match:
+                        return p_match.group(1) == g_match.group(1)
+                    return g_str in p_str
+                
+                tasks.append(mcq_check(pred, gt))
+            else:
+                gt = data[idx].get("ground_truth_answer", "")
+                tasks.append(math_utils_v.is_equal(pred, gt, executor))
+                
+        return await asyncio.gather(*tasks)
+
+    correctness_all = asyncio.run(get_correctness_all())
+    for p, ok in enumerate(correctness_all):
+        index_i = index_list[p]
+        data[index_i]["correctness"].append(bool(ok))
+
+    # =========================================================================
+    # 核心新增逻辑：跨样本群组计算 T-GRPO 奖励 和 逆转无偏奖励
+    # =========================================================================
+    
+    # 1. 按照 data_id 分组映射索引
+    grouped_data = defaultdict(lambda: {
+        "forward_False": [], "backward_False": [], 
+        "forward_True": [],  "backward_True": []
+    })
+
+    for i in range(len(data)):
+        d_id = data[i].get("data_id")
+        if d_id is None:
+            continue
+        r_type = data[i].get("reversal_type", "forward")
+        is_shuf = str(data[i].get("is_shuffled", False))
+        grouped_data[d_id][f"{r_type}_{is_shuf}"].append(i)
+
+    # 计算并应用群组奖励
+    all_temporal_bonuses = []  # 新增：记录所有的时序红利
+    all_unbiased_bonuses = []  # 新增：记录所有的无偏红利
+
+    # 2. 计算并应用群组奖励
+    for d_id, group in grouped_data.items():
+        
+        def get_acc(key):
+            if not group[key]: return 0.0
+            idx = group[key][0]
+            corr = data[idx]["correctness"]
+            return sum(corr) / len(corr) if len(corr) > 0 else 0.0
+
+        # 获取四组情况的正确率
+        acc_f_ord = get_acc("forward_False")
+        acc_f_shuf = get_acc("forward_True")
+        acc_b_ord = get_acc("backward_False")
+        acc_b_shuf = get_acc("backward_True")
+
+        # T-GRPO 奖励: 顺序正确率 >= 0.8 * 打乱正确率，得 0.3
+        f_temporal_bonus = 0.3 if acc_f_ord >= 0.8 * acc_f_shuf else 0.0
+        b_temporal_bonus = 0.3 if acc_b_ord >= 0.8 * acc_b_shuf else 0.0
+
+        # 逆转无偏奖励: 正反正确率差值 <= 0.2，且都不是彻底全错，得 0.3
+        unbiased_bonus = 0.0
+        if abs(acc_f_ord - acc_b_ord) <= 0.2 and (acc_f_ord > 0.1 or acc_b_ord > 0.1):
+            unbiased_bonus = 0.3
+
+        # 新增：将当前组的奖励记录下来用于统计
+        if group["forward_False"]:
+            all_temporal_bonuses.append(f_temporal_bonus)
+            all_unbiased_bonuses.append(unbiased_bonus)
+        if group["backward_False"]:
+            all_temporal_bonuses.append(b_temporal_bonus)
+            all_unbiased_bonuses.append(unbiased_bonus)
+
+        # 仅将奖励发放给 顺序组(is_shuffled=False) 中回答正确的样本
+        def apply_bonus(key, temporal_bonus, unbiased_bonus):
+            if not group[key]: return
+            idx = group[key][0]
+            raw_rewards = []
+            for is_correct in data[idx]["correctness"]:
+                if is_correct:
+                    # 基础分 1.0 + 时序红利 + 无偏红利
+                    raw_rewards.append(1.0 + temporal_bonus + unbiased_bonus)
+                else:
+                    raw_rewards.append(0.0)
+            data[idx]["raw_rewards"] = raw_rewards
+
+        apply_bonus("forward_False", f_temporal_bonus, unbiased_bonus)
+        apply_bonus("backward_False", b_temporal_bonus, unbiased_bonus)
+
+    # =========================================================================
+
+    def z_score_normalize(lst):
+        mean = sum(lst) / len(lst) if len(lst) > 0 else 0.0
+        std = (sum((x - mean) ** 2 for x in lst) / len(lst)) ** 0.5 if len(lst) > 0 else 0.0
+        if std == 0:
+            return [0 for _ in lst]
+        return [(x - mean) / std for x in lst]
+
+    final_data = []
+    ordered_correctness = [] # 用于日志打印，不记录被打乱的实验数据
+    
+    for i in range(len(data)):
+        # 丢弃打乱的对照组，它们仅仅用于上面的计算参照，不进入强化学习
+        if data[i].get("is_shuffled", False):
+            continue
+            
+        ordered_correctness.extend(data[i]["correctness"])
+
+        # 如果部分数据没有走过分组逻辑，默认回退为原始 correct 分数
+        raw_rewards = data[i].get("raw_rewards", [float(c) for c in data[i]["correctness"]])
+        lengths = data[i]["response_length"]
+        
+        # 长度惩罚
+        if config.reward.strict_len_check:
+            for j in range(len(lengths)):
+                if OmegaConf.select(config, "rollout.max_gen_length", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_gen_length - 5:
+                    raw_rewards[j] = 0.0
+                if OmegaConf.select(config, "rollout.max_token", default=MISSING) is not MISSING and lengths[j] >= config.rollout.max_token - 5:
+                    raw_rewards[j] = 0.0
+
+        # Z-Score 优势归一化
+        rewards = z_score_normalize(raw_rewards)
+        data[i]["rewards"] = rewards
+
+        if config.experiment.function == "train":
+            proportion = sum(data[i]["correctness"]) / len(data[i]["correctness"]) if len(data[i]["correctness"]) > 0 else 0.0
+            # 如果全部回答都是错的，或者全是对的（无差别区分度），则剔除该样本
+            if proportion > 0.95 or proportion < 0.05:
+                continue
+
+            for j in range(len(rewards)):
+                data_i = {}
+                data_i["prompt"] = data[i]["prompt"]
+                data_i["reward"] = rewards[j]
+                data_i["response"] = data[i]["full_output"][j]
+                data_i["step_map"] = data[i]["step_map"][j]
+                
+                # 保留其他需要的上下文 ID
+                data_i["data_id"] = data[i].get("data_id")
+                data_i["reversal_type"] = data[i].get("reversal_type")
+                
+                if "image_token_ids" in data[i]:
+                    data_i["image_token_ids"] = data[i]["image_token_ids"]
+                
+                p_video_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "video")
+                if p_video_abs is not None:
+                    data_i["video_abs_path"] = p_video_abs
+                
+                p_image_abs = resolve_media_abs_path(data[i], config.dataset.get("image_root", None), "image")
+                if p_image_abs is not None:
+                    data_i["image_abs_path"] = p_image_abs
+
+                final_data.append(data_i)
+
+        if config.experiment.function == "evaluation":
+            data[i]["step_map"] = []
+
+    if config.experiment.function == "train":
+        with open("../" + project_name + "/temp_data/" + config.dataset.optimization_data + ".json", "w", encoding="utf-8") as f:
+            json.dump(final_data, f, indent=2, ensure_ascii=False)
+
+
+    os.makedirs(os.path.dirname(file_name), exist_ok=True)
+    with open(file_name, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+
+
+    outputs_result_name = "../" + project_name + "/results/results-" + outputs_name + ".txt"
+    os.makedirs(os.path.dirname(outputs_result_name), exist_ok=True)
+    with open(outputs_result_name, "a") as f:
+        # Save + print
+        def save_and_print(text):
+            cprint("\n\n\n" + text, color="green")
+            f.write(text + "\n")
+        
+        # 只计算有序组（有效学习组）的准确率来作为监控指标
+        acc = (sum(ordered_correctness)/len(ordered_correctness)) if len(ordered_correctness) > 0 else float("nan")
+        avg_len = sum(response_length_list)/len(response_length_list)
+
+        # 新增：计算 T-GRPO 和 Unbiased 奖励的平均值
+        avg_temp = sum(all_temporal_bonuses)/len(all_temporal_bonuses) if all_temporal_bonuses else 0.0
+        avg_unbias = sum(all_unbiased_bonuses)/len(all_unbiased_bonuses) if all_unbiased_bonuses else 0.0
+
+        output_text = f"train step: {config.experiment.current_epoch}  "
+        if config.experiment.function == "train":
+            if config.model.model_base != "sdar" and config.model.model_base != "trado":
+                output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy}  block_size: {config.rollout.block_size}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+            else:
+                output_text = output_text + f"remasking_strategy: {config.rollout.remasking_strategy}  top_k: {config.rollout.top_k}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+        else:
+            if config.model.model_base != "sdar" and config.model.model_base != "trado":
+                output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy}  block_size: {config.evaluation.block_size}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+            else:
+                output_text = output_text + f"remasking_strategy: {config.evaluation.remasking_strategy}  top_k: {config.evaluation.top_k}  acc: {acc:.4f}  avg length: {avg_len:.2f}"
+        save_and_print(output_text)
+
+    if config.experiment.function == "train":
+        csv_file_path = f"../{project_name}/results/all_rewards.csv"
+        
+        # 如果是第1个 epoch/step，或者是新文件，就先写入表头
+        write_header = not os.path.exists(csv_file_path) or config.experiment.current_epoch == 1
+        
+        with open(csv_file_path, "a", encoding="utf-8") as f_csv:
+            if write_header:
+                f_csv.write("step,acc_reward,t_grpo_reward,unbiased_reward,response_length\n")
+            
+            # 按行追加当前 step 的数据
+            f_csv.write(f"{config.experiment.current_epoch},{acc:.4f},{avg_temp:.4f},{avg_unbias:.4f},{avg_len:.2f}\n")
+
+        if run is not None:
+            wandb.log(
+                {
+                    "reward/acc": acc,
+                    "reward/t_grpo_bonus": avg_temp,
+                    "reward/unbiased_bonus": avg_unbias,
+                    "reward/avg_response_length": avg_len,
+                    "reward/train_step": config.experiment.current_epoch,
+                },
+                step=config.experiment.current_epoch,
+            )
+            wandb.finish()

reward/wandb/run-20260530_063905-fmmkx1a5/files/config.yaml

@@ -0,0 +1,281 @@
+_wandb:
+    value:
+        cli_version: 0.25.0
+        code_path: code/reward/rl_reward_video_reversal.py
+        e:
+            cabjrf3uennqcbtqsnxq22932fx3o8bo:
+                args:
+                    - config=../configs/rl_lavida_video_reversal.yaml
+                    - wandb.run_id=fmmkx1a5
+                    - experiment.function=train
+                    - evaluation.block_size=None
+                    - evaluation.top_k=None
+                    - evaluation.remasking_strategy=None
+                    - experiment.current_epoch=10
+                codePath: reward/rl_reward_video_reversal.py
+                codePathLocal: rl_reward_video_reversal.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "806578724864"
+                        used: "625483096064"
+                email: shixi_runqing@staff.weibo.com
+                executable: /data8/runqing/miniconda/envs/dllm-rl/bin/python
+                git:
+                    commit: 10b4fd1cba83e4496c20585dc2fd01dc02f20255
+                    remote: https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git
+                gpu: NVIDIA H800
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-f768b684-b9ce-23a2-81a6-af16161cf9be
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-f531028f-805c-3301-61b3-0c46329dc9c0
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-2775489f-18c8-2328-8829-c46238575379
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H800
+                      uuid: GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd
+                host: b1301b38c75f
+                memory:
+                    total: "2164113764352"
+                os: Linux-5.15.0-144-generic-x86_64-with-glibc2.35
+                program: /data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py
+                python: CPython 3.10.19
+                root: /data4/runqing/dLLM-RL/reward
+                startedAt: "2026-05-29T22:39:05.986619Z"
+                writerId: cabjrf3uennqcbtqsnxq22932fx3o8bo
+        m: []
+        python_version: 3.10.19
+        t:
+            "1":
+                - 1
+                - 41
+            "2":
+                - 1
+                - 41
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 16
+                - 61
+                - 62
+            "4": 3.10.19
+            "5": 0.25.0
+            "10":
+                - 20
+            "12": 0.25.0
+            "13": linux-x86_64
+config:
+    value: ../configs/rl_lavida_video_reversal.yaml
+dataset.data_type:
+    value: video_reverse
+dataset.optimization_data:
+    value: rl_data
+dataset.train_dataset:
+    value: video_reversal_rl_train
+evaluation.batch_size:
+    value: 1
+evaluation.block_size:
+    value: None
+evaluation.data_type:
+    value: video_reverse
+evaluation.dynamic_threshold:
+    value: 0.9
+evaluation.eval_dataset:
+    value: video_reversal_rl_eval
+evaluation.further_horizon:
+    value: 128
+evaluation.if_eval:
+    value: false
+evaluation.max_gen_length:
+    value: 128
+evaluation.num_response_per_task:
+    value: 3
+evaluation.remasking_strategy:
+    value: None
+evaluation.remasking_strategy.0:
+    value: low_confidence_static
+evaluation.remasking_strategy.1:
+    value: low_confidence_dynamic
+evaluation.steps:
+    value: 128
+evaluation.target:
+    value: confidence
+evaluation.temperature:
+    value: 0.1
+evaluation.top_k:
+    value: None
+evaluation.use_cache:
+    value: true
+execute.num_chunk:
+    value: 128
+experiment.current_epoch:
+    value: 10
+experiment.deepspeed_file:
+    value: 1_node_8_gpus_deepspeed_zero2
+experiment.eval_every:
+    value: 5
+experiment.function:
+    value: train
+experiment.logging_dir:
+    value: rl_lavida_video_reversal/logs
+experiment.node_index:
+    value: 0
+experiment.num_node:
+    value: 1
+experiment.project:
+    value: rl_lavida_video_reversal
+experiment.save_every:
+    value: 10
+experiment.start_from_scratch:
+    value: true
+experiment.total_step:
+    value: 30
+lr_scheduler.params.learning_rate:
+    value: 1e-06
+lr_scheduler.params.min_lr_scale:
+    value: 1
+lr_scheduler.params.warmup_steps:
+    value: 0
+lr_scheduler.scheduler:
+    value: cosine
+model.image_resolution:
+    value: 512
+model.model_base:
+    value: lavida
+model.optimized_name:
+    value: optimized
+model.pretrained_model:
+    value: /data4/runqing/dLLM-RL/sft_lavida_video_reversal/ckpt/optimized
+model.vq_model_path:
+    value: /data4/runqing/hf/magvitv2
+optimizer.name:
+    value: adamw
+optimizer.params.beta1:
+    value: 0.9
+optimizer.params.beta2:
+    value: 0.999
+optimizer.params.epsilon:
+    value: 1e-08
+optimizer.params.learning_rate:
+    value: 1e-06
+optimizer.params.scale_lr:
+    value: false
+optimizer.params.weight_decay:
+    value: 0
+reward.answer_must_in_box:
+    value: true
+reward.strict_len_check:
+    value: false
+rollout.batch_size:
+    value: 1
+rollout.block_size:
+    value: 32
+rollout.dynamic_threshold:
+    value: 0.9
+rollout.further_horizon:
+    value: 128
+rollout.max_gen_length:
+    value: 128
+rollout.num_response_per_task:
+    value: 8
+rollout.num_task_per_step:
+    value: 128
+rollout.remasking_strategy:
+    value: low_confidence_dynamic
+rollout.steps:
+    value: 128
+rollout.target:
+    value: confidence
+rollout.temperature:
+    value: 0.8
+rollout.use_cache:
+    value: true
+system.HF_HOME:
+    value: null
+system.HTTP_PROXY:
+    value: null
+system.base_dir:
+    value: /data4/runqing/dLLM-RL
+system.env_name:
+    value: /data8/runqing/miniconda/envs/dllm-rl
+system.envs_dir:
+    value: /data8/runqing/miniconda/envs/dllm-rl
+training.batch_size_lm:
+    value: 1
+training.beta:
+    value: 0.01
+training.enable_tf32:
+    value: true
+training.eps:
+    value: 0.2
+training.gradient_accumulation_steps:
+    value: 2
+training.lower_p:
+    value: 0.35
+training.mask_times_per_sample:
+    value: 2
+training.max_gen_length:
+    value: 128
+training.max_grad_norm:
+    value: 1
+training.max_prompt_len:
+    value: 512
+training.method:
+    value: random_masking
+training.mixed_precision:
+    value: bf16
+training.num_train_epochs:
+    value: 1
+training.post_num:
+    value: 1
+training.seed:
+    value: 10086
+training.shrink:
+    value: 8
+training.upper_p:
+    value: 0.9
+training.use_kl_estimator_k3:
+    value: true
+wandb.entity:
+    value: null
+wandb.resume:
+    value: auto
+wandb.run_id:
+    value: fmmkx1a5

reward/wandb/run-20260530_063905-fmmkx1a5/files/output.log

@@ -0,0 +1,4 @@
+
+
+
+train step: 10  remasking_strategy: low_confidence_dynamic  block_size: 32  acc: 0.5078  avg length: 125.81

reward/wandb/run-20260530_063905-fmmkx1a5/files/requirements.txt

@@ -0,0 +1,157 @@
+transformers==4.52.4
+nvidia-nvtx-cu12==12.4.127
+deepspeed==0.18.6
+typing-inspection==0.4.2
+scipy==1.15.3
+MarkupSafe==3.0.3
+nvidia-cuda-runtime-cu12==12.4.127
+detoxify==0.5.2
+yarl==1.22.0
+tornado==6.5.4
+nvidia-curand-cu12==10.3.5.147
+nest-asyncio==1.6.0
+safetensors==0.7.0
+async-timeout==5.0.1
+typing_extensions==4.15.0
+asttokens==3.0.1
+cuda-python==13.1.1
+executing==2.2.1
+nvidia-cudnn-frontend==1.18.0
+comm==0.2.3
+tokenizers==0.21.4
+six==1.17.0
+Pygments==2.19.2
+nvidia-cusparselt-cu12==0.6.2
+apache-tvm-ffi==0.1.8.post2
+flash_attn==2.7.4.post1
+tqdm==4.67.3
+tensorboard-data-server==0.7.2
+annotated-types==0.7.0
+math-verify==0.9.0
+idna==3.11
+regex==2026.1.15
+nvidia-cutlass-dsl-libs-base==4.4.0
+flashinfer-python==0.6.3
+click==8.3.1
+pytz==2025.2
+wheel==0.46.3
+ipykernel==7.2.0
+stack-data==0.6.3
+absl-py==2.4.0
+wandb==0.25.0
+pandas==2.3.3
+termcolor==3.3.0
+sympy==1.13.1
+joblib==1.5.3
+prompt_toolkit==3.0.52
+attrs==25.4.0
+nvidia-cusolver-cu12==11.6.1.9
+xxhash==3.6.0
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+frozenlist==1.8.0
+nvidia-ml-py==13.590.48
+multiprocess==0.70.18
+decorator==5.2.1
+cuda-pathfinder==1.3.4
+multidict==6.7.1
+pure_eval==0.2.3
+nvidia-cufft-cu12==11.2.1.3
+omegaconf==2.3.0
+PyYAML==6.0.3
+debugpy==1.8.20
+psutil==7.2.2
+ninja==1.13.0
+nvidia-cublas-cu12==12.4.5.8
+triton==3.2.0
+antlr4-python3-runtime==4.9.3
+peft==0.18.1
+aiosignal==1.4.0
+accelerate==1.12.0
+liger_kernel==0.7.0
+anyio==4.12.1
+decord==0.6.0
+h11==0.16.0
+urllib3==2.6.3
+wcwidth==0.6.0
+networkx==3.4.2
+huggingface_hub==0.36.2
+tzdata==2025.3
+xformers==0.0.29.post3
+tabulate==0.9.0
+certifi==2026.1.4
+tensorboard==2.20.0
+matplotlib-inline==0.2.1
+pip==26.0.1
+smmap==5.0.2
+nvidia-cuda-cupti-cu12==12.4.127
+Markdown==3.10.2
+nvidia-nccl-cu12==2.21.5
+aiohttp==3.13.3
+scikit-learn==1.7.2
+nvidia-cutlass-dsl==4.4.0
+setuptools==82.0.0
+jupyter_core==5.9.1
+asyncio==4.0.0
+einops==0.8.2
+pydantic==2.12.5
+av==16.1.0
+protobuf==6.33.5
+evaluate==0.4.6
+Jinja2==3.1.6
+exceptiongroup==1.3.1
+ipython==8.38.0
+sentry-sdk==2.53.0
+httpx==0.28.1
+msgpack==1.1.2
+platformdirs==4.9.2
+hjson==3.1.0
+torchvision==0.21.0
+parso==0.8.6
+cuda-bindings==13.1.1
+pexpect==4.9.0
+threadpoolctl==3.6.0
+pyarrow==23.0.1
+jedi==0.19.2
+nvidia-cuda-nvrtc-cu12==12.4.127
+Werkzeug==3.1.5
+fsspec==2025.10.0
+pydantic_core==2.41.5
+GitPython==3.1.46
+propcache==0.4.1
+mpmath==1.3.0
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-cusparse-cu12==12.3.1.170
+hf-xet==1.2.0
+charset-normalizer==3.4.4
+dill==0.4.0
+datasets==4.5.0
+py-cpuinfo==9.0.0
+sentencepiece==0.2.1
+latex2sympy2_extended==1.11.0
+jupyter_client==8.8.0
+pyzmq==27.1.0
+numpy==2.2.6
+gitdb==4.0.12
+grpcio==1.78.0
+packaging==26.0
+requests==2.32.5
+nvidia-cudnn-cu12==9.1.0.70
+httpcore==1.0.9
+traitlets==5.14.3
+filelock==3.24.2
+ptyprocess==0.7.0
+pillow==12.1.1
+torch==2.6.0
+more-itertools==10.8.0
+wheel==0.46.3
+jaraco.text==4.0.0
+zipp==3.23.0
+autocommand==2.2.2
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+tomli==2.4.0
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+platformdirs==4.4.0
+packaging==26.0

reward/wandb/run-20260530_063905-fmmkx1a5/files/wandb-metadata.json

@@ -0,0 +1,98 @@
+{
+  "os":  "Linux-5.15.0-144-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.19",
+  "startedAt":  "2026-05-29T22:39:05.986619Z",
+  "args":  [
+    "config=../configs/rl_lavida_video_reversal.yaml",
+    "wandb.run_id=fmmkx1a5",
+    "experiment.function=train",
+    "evaluation.block_size=None",
+    "evaluation.top_k=None",
+    "evaluation.remasking_strategy=None",
+    "experiment.current_epoch=10"
+  ],
+  "program":  "/data4/runqing/dLLM-RL/reward/rl_reward_video_reversal.py",
+  "codePath":  "reward/rl_reward_video_reversal.py",
+  "codePathLocal":  "rl_reward_video_reversal.py",
+  "git":  {
+    "remote":  "https://gitclone.com/github.com/Gen-Verse/dLLM-RL.git",
+    "commit":  "10b4fd1cba83e4496c20585dc2fd01dc02f20255"
+  },
+  "email":  "shixi_runqing@staff.weibo.com",
+  "root":  "/data4/runqing/dLLM-RL/reward",
+  "host":  "b1301b38c75f",
+  "executable":  "/data8/runqing/miniconda/envs/dllm-rl/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H800",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "806578724864",
+      "used":  "625483096064"
+    }
+  },
+  "memory":  {
+    "total":  "2164113764352"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f768b684-b9ce-23a2-81a6-af16161cf9be"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2f1e1a7b-76a2-e00b-3dc5-79add059b8f9"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-33d7b8d4-af9c-4719-9686-1712e9694cd6"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0b797f66-23ec-d0ff-1c65-fb34b9064e53"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-ad9e9c73-0f31-a8b3-e8f7-7be19f5338ad"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f531028f-805c-3301-61b3-0c46329dc9c0"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2775489f-18c8-2328-8829-c46238575379"
+    },
+    {
+      "name":  "NVIDIA H800",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1dd27781-c1f3-d980-6f18-fe8a778b69fd"
+    }
+  ],
+  "cudaVersion":  "12.8",
+  "writerId":  "cabjrf3uennqcbtqsnxq22932fx3o8bo"
+}