qwen3_1p7b_base_mbpp_nobuf_lr6e7 logs

qwen3_1p7b_base_mbpp_nobuf_lr6e7/logs/train.log

@@ -0,0 +1,635 @@
+/scratch/js15262/rl-forgetting-experiments/reward-signal-analysis/verl/__init__.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
+  import pkg_resources
+2026-11-01 11:13:52,258	INFO worker.py:1908 -- Started a local Ray instance. View the dashboard at http://127.0.0.1:8265 
+(pid=56225) /scratch/js15262/rl-forgetting-experiments/reward-signal-analysis/verl/__init__.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
+(pid=56225)   import pkg_resources
+(TaskRunner pid=56225) TaskRunner hostname: gh129.hpc.nyu.edu, PID: 56225
+(TaskRunner pid=56225) {'actor_rollout_ref': {'actor': {'buffer': {'ce_threshold': None,
+(TaskRunner pid=56225)                                             'cooldown_step': 20,
+(TaskRunner pid=56225)                                             'drop_zero_weight': True,
+(TaskRunner pid=56225)                                             'enable': False,
+(TaskRunner pid=56225)                                             'group_admission': False,
+(TaskRunner pid=56225)                                             'hard_k': 32,
+(TaskRunner pid=56225)                                             'initial_rollout_priority': 'max',
+(TaskRunner pid=56225)                                             'lambda': 0.0,
+(TaskRunner pid=56225)                                             'lambda_anneal_steps': 0,
+(TaskRunner pid=56225)                                             'lambda_end': 0.0,
+(TaskRunner pid=56225)                                             'lambda_epsilon': 0.0001,
+(TaskRunner pid=56225)                                             'lambda_mode': 'static',
+(TaskRunner pid=56225)                                             'lambda_rho': 1.0,
+(TaskRunner pid=56225)                                             'loss_mode': 'sft',
+(TaskRunner pid=56225)                                             'rollout_score_beta': 1.0,
+(TaskRunner pid=56225)                                             'rollout_score_tau': 1.0,
+(TaskRunner pid=56225)                                             'sample_size': 2048,
+(TaskRunner pid=56225)                                             'sample_strategy': 'all',
+(TaskRunner pid=56225)                                             'size': -1,
+(TaskRunner pid=56225)                                             'softmax_prompt_mode': 'balanced',
+(TaskRunner pid=56225)                                             'threshold': 0.0},
+(TaskRunner pid=56225)                                  'calculate_entropy_for_old_log_prob': True,
+(TaskRunner pid=56225)                                  'checkpoint': {'load_contents': ['model',
+(TaskRunner pid=56225)                                                                   'optimizer',
+(TaskRunner pid=56225)                                                                   'extra'],
+(TaskRunner pid=56225)                                                 'save_contents': ['model',
+(TaskRunner pid=56225)                                                                   'optimizer',
+(TaskRunner pid=56225)                                                                   'extra']},
+(TaskRunner pid=56225)                                  'clip_ratio': 0.2,
+(TaskRunner pid=56225)                                  'clip_ratio_c': 3.0,
+(TaskRunner pid=56225)                                  'clip_ratio_high': 0.2,
+(TaskRunner pid=56225)                                  'clip_ratio_low': 0.2,
+(TaskRunner pid=56225)                                  'entropy_checkpointing': False,
+(TaskRunner pid=56225)                                  'entropy_coeff': 0.0,
+(TaskRunner pid=56225)                                  'entropy_from_logits_with_chunking': False,
+(TaskRunner pid=56225)                                  'fsdp_config': {'forward_prefetch': False,
+(TaskRunner pid=56225)                                                  'fsdp_size': -1,
+(TaskRunner pid=56225)                                                  'offload_policy': False,
+(TaskRunner pid=56225)                                                  'optimizer_offload': False,
+(TaskRunner pid=56225)                                                  'param_offload': False,
+(TaskRunner pid=56225)                                                  'reshard_after_forward': True,
+(TaskRunner pid=56225)                                                  'wrap_policy': {'min_num_params': 0}},
+(TaskRunner pid=56225)                                  'grad_clip': 1.0,
+(TaskRunner pid=56225)                                  'kl_loss_coef': 0.0,
+(TaskRunner pid=56225)                                  'kl_loss_type': 'low_var_kl',
+(TaskRunner pid=56225)                                  'loss_agg_mode': 'token-mean',
+(TaskRunner pid=56225)                                  'optim': {'lr': 6e-07,
+(TaskRunner pid=56225)                                            'lr_warmup_steps': -1,
+(TaskRunner pid=56225)                                            'lr_warmup_steps_ratio': 0.0,
+(TaskRunner pid=56225)                                            'min_lr_ratio': 0.0,
+(TaskRunner pid=56225)                                            'num_cycles': 0.5,
+(TaskRunner pid=56225)                                            'total_training_steps': -1,
+(TaskRunner pid=56225)                                            'warmup_style': 'constant',
+(TaskRunner pid=56225)                                            'weight_decay': 0.01},
+(TaskRunner pid=56225)                                  'policy_loss': {'clip_cov_lb': 1.0,
+(TaskRunner pid=56225)                                                  'clip_cov_ratio': 0.0002,
+(TaskRunner pid=56225)                                                  'clip_cov_ub': 5.0,
+(TaskRunner pid=56225)                                                  'kl_cov_ratio': 0.0002,
+(TaskRunner pid=56225)                                                  'loss_mode': 'vanilla',
+(TaskRunner pid=56225)                                                  'ppo_kl_coef': 0.1},
+(TaskRunner pid=56225)                                  'ppo_epochs': 1,
+(TaskRunner pid=56225)                                  'ppo_max_token_len_per_gpu': 16384,
+(TaskRunner pid=56225)                                  'ppo_micro_batch_size': None,
+(TaskRunner pid=56225)                                  'ppo_micro_batch_size_per_gpu': 8,
+(TaskRunner pid=56225)                                  'ppo_mini_batch_size': 64,
+(TaskRunner pid=56225)                                  'profiler': {'all_ranks': False,
+(TaskRunner pid=56225)                                               'discrete': False,
+(TaskRunner pid=56225)                                               'ranks': None},
+(TaskRunner pid=56225)                                  'shuffle': False,
+(TaskRunner pid=56225)                                  'strategy': 'fsdp',
+(TaskRunner pid=56225)                                  'ulysses_sequence_parallel_size': 1,
+(TaskRunner pid=56225)                                  'use_dynamic_bsz': False,
+(TaskRunner pid=56225)                                  'use_kl_loss': False,
+(TaskRunner pid=56225)                                  'use_torch_compile': True},
+(TaskRunner pid=56225)                        'hybrid_engine': True,
+(TaskRunner pid=56225)                        'model': {'custom_chat_template': None,
+(TaskRunner pid=56225)                                  'enable_activation_offload': False,
+(TaskRunner pid=56225)                                  'enable_gradient_checkpointing': True,
+(TaskRunner pid=56225)                                  'exclude_modules': None,
+(TaskRunner pid=56225)                                  'external_lib': None,
+(TaskRunner pid=56225)                                  'fused_kernel_options': {'impl_backend': 'torch'},
+(TaskRunner pid=56225)                                  'lora_alpha': 16,
+(TaskRunner pid=56225)                                  'lora_rank': 0,
+(TaskRunner pid=56225)                                  'override_config': {},
+(TaskRunner pid=56225)                                  'path': '/scratch/js15262/.huggingface/hub/models--Qwen--Qwen3-1.7B-Base/snapshots/ea980cb0a6c2ae4b936e82123acc929f1cec04c1',
+(TaskRunner pid=56225)                                  'target_modules': 'all-linear',
+(TaskRunner pid=56225)                                  'trust_remote_code': False,
+(TaskRunner pid=56225)                                  'use_fused_kernels': False,
+(TaskRunner pid=56225)                                  'use_liger': False,
+(TaskRunner pid=56225)                                  'use_remove_padding': True,
+(TaskRunner pid=56225)                                  'use_shm': False},
+(TaskRunner pid=56225)                        'ref': {'entropy_checkpointing': False,
+(TaskRunner pid=56225)                                'entropy_from_logits_with_chunking': False,
+(TaskRunner pid=56225)                                'fsdp_config': {'forward_prefetch': False,
+(TaskRunner pid=56225)                                                'param_offload': True,
+(TaskRunner pid=56225)                                                'reshard_after_forward': True,
+(TaskRunner pid=56225)                                                'wrap_policy': {'min_num_params': 0}},
+(TaskRunner pid=56225)                                'log_prob_max_token_len_per_gpu': 16384,
+(TaskRunner pid=56225)                                'log_prob_micro_batch_size': None,
+(TaskRunner pid=56225)                                'log_prob_micro_batch_size_per_gpu': 8,
+(TaskRunner pid=56225)                                'log_prob_use_dynamic_bsz': False,
+(TaskRunner pid=56225)                                'profiler': {'all_ranks': False,
+(TaskRunner pid=56225)                                             'discrete': False,
+(TaskRunner pid=56225)                                             'ranks': None},
+(TaskRunner pid=56225)                                'strategy': 'fsdp',
+(TaskRunner pid=56225)                                'ulysses_sequence_parallel_size': 1,
+(TaskRunner pid=56225)                                'use_torch_compile': True},
+(TaskRunner pid=56225)                        'rollout': {'agent': {'num_workers': 8},
+(TaskRunner pid=56225)                                    'calculate_log_probs': False,
+(TaskRunner pid=56225)                                    'disable_log_stats': True,
+(TaskRunner pid=56225)                                    'do_sample': True,
+(TaskRunner pid=56225)                                    'dtype': 'bfloat16',
+(TaskRunner pid=56225)                                    'enable_chunked_prefill': True,
+(TaskRunner pid=56225)                                    'enforce_eager': True,
+(TaskRunner pid=56225)                                    'engine_kwargs': {'sglang': {'attention_backend': None},
+(TaskRunner pid=56225)                                                      'vllm': {'disable_mm_preprocessor_cache': False,
+(TaskRunner pid=56225)                                                               'swap_space': None}},
+(TaskRunner pid=56225)                                    'free_cache_engine': True,
+(TaskRunner pid=56225)                                    'gpu_memory_utilization': 0.7,
+(TaskRunner pid=56225)                                    'ignore_eos': False,
+(TaskRunner pid=56225)                                    'layered_summon': False,
+(TaskRunner pid=56225)                                    'load_format': 'dummy_dtensor',
+(TaskRunner pid=56225)                                    'log_prob_max_token_len_per_gpu': 16384,
+(TaskRunner pid=56225)                                    'log_prob_micro_batch_size': None,
+(TaskRunner pid=56225)                                    'log_prob_micro_batch_size_per_gpu': 8,
+(TaskRunner pid=56225)                                    'log_prob_use_dynamic_bsz': False,
+(TaskRunner pid=56225)                                    'max_model_len': None,
+(TaskRunner pid=56225)                                    'max_num_batched_tokens': 4096,
+(TaskRunner pid=56225)                                    'max_num_seqs': 1024,
+(TaskRunner pid=56225)                                    'mode': 'sync',
+(TaskRunner pid=56225)                                    'multi_stage_wake_up': False,
+(TaskRunner pid=56225)                                    'multi_turn': {'completion_callback': None,
+(TaskRunner pid=56225)                                                   'enable': False,
+(TaskRunner pid=56225)                                                   'format': 'hermes',
+(TaskRunner pid=56225)                                                   'interaction_config_path': None,
+(TaskRunner pid=56225)                                                   'max_assistant_turns': None,
+(TaskRunner pid=56225)                                                   'max_parallel_calls': 1,
+(TaskRunner pid=56225)                                                   'max_tool_response_length': 256,
+(TaskRunner pid=56225)                                                   'max_user_turns': None,
+(TaskRunner pid=56225)                                                   'tokenization_sanity_check_mode': 'strict',
+(TaskRunner pid=56225)                                                   'tool_config_path': None,
+(TaskRunner pid=56225)                                                   'tool_response_truncate_side': 'middle',
+(TaskRunner pid=56225)                                                   'use_inference_chat_template': False},
+(TaskRunner pid=56225)                                    'n': 8,
+(TaskRunner pid=56225)                                    'name': 'vllm',
+(TaskRunner pid=56225)                                    'profiler': {'all_ranks': False,
+(TaskRunner pid=56225)                                                 'discrete': False,
+(TaskRunner pid=56225)                                                 'ranks': None},
+(TaskRunner pid=56225)                                    'prompt_length': 1024,
+(TaskRunner pid=56225)                                    'response_length': 3072,
+(TaskRunner pid=56225)                                    'temperature': 1.0,
+(TaskRunner pid=56225)                                    'tensor_model_parallel_size': 1,
+(TaskRunner pid=56225)                                    'top_k': -1,
+(TaskRunner pid=56225)                                    'top_p': 1,
+(TaskRunner pid=56225)                                    'val_kwargs': {'do_sample': False,
+(TaskRunner pid=56225)                                                   'n': 1,
+(TaskRunner pid=56225)                                                   'temperature': 0,
+(TaskRunner pid=56225)                                                   'top_k': -1,
+(TaskRunner pid=56225)                                                   'top_p': 1.0}}},
+(TaskRunner pid=56225)  'algorithm': {'adv_estimator': 'grpo',
+(TaskRunner pid=56225)                'gamma': 1.0,
+(TaskRunner pid=56225)                'hist_baseline': {'k': 64, 'min_count': 8},
+(TaskRunner pid=56225)                'kl_ctrl': {'horizon': 10000,
+(TaskRunner pid=56225)                            'kl_coef': 0.001,
+(TaskRunner pid=56225)                            'target_kl': 0.1,
+(TaskRunner pid=56225)                            'type': 'fixed'},
+(TaskRunner pid=56225)                'kl_penalty': 'kl',
+(TaskRunner pid=56225)                'lam': 1.0,
+(TaskRunner pid=56225)                'norm_adv_by_std_in_grpo': False,
+(TaskRunner pid=56225)                'pf_ppo': {'reweight_method': 'pow', 'weight_pow': 2.0},
+(TaskRunner pid=56225)                'use_kl_in_reward': False,
+(TaskRunner pid=56225)                'use_pf_ppo': False},
+(TaskRunner pid=56225)  'critic': {'checkpoint': {'load_contents': ['model', 'optimizer', 'extra'],
+(TaskRunner pid=56225)                            'save_contents': ['model', 'optimizer', 'extra']},
+(TaskRunner pid=56225)             'cliprange_value': 0.5,
+(TaskRunner pid=56225)             'forward_max_token_len_per_gpu': 32768,
+(TaskRunner pid=56225)             'forward_micro_batch_size': None,
+(TaskRunner pid=56225)             'forward_micro_batch_size_per_gpu': None,
+(TaskRunner pid=56225)             'grad_clip': 1.0,
+(TaskRunner pid=56225)             'loss_agg_mode': 'token-mean',
+(TaskRunner pid=56225)             'model': {'enable_activation_offload': False,
+(TaskRunner pid=56225)                       'enable_gradient_checkpointing': True,
+(TaskRunner pid=56225)                       'external_lib': None,
+(TaskRunner pid=56225)                       'fsdp_config': {'forward_prefetch': False,
+(TaskRunner pid=56225)                                       'fsdp_size': -1,
+(TaskRunner pid=56225)                                       'offload_policy': False,
+(TaskRunner pid=56225)                                       'optimizer_offload': False,
+(TaskRunner pid=56225)                                       'param_offload': False,
+(TaskRunner pid=56225)                                       'reshard_after_forward': True,
+(TaskRunner pid=56225)                                       'wrap_policy': {'min_num_params': 0}},
+(TaskRunner pid=56225)                       'lora_alpha': 16,
+(TaskRunner pid=56225)                       'lora_rank': 0,
+(TaskRunner pid=56225)                       'override_config': {},
+(TaskRunner pid=56225)                       'path': '~/models/deepseek-llm-7b-chat',
+(TaskRunner pid=56225)                       'target_modules': 'all-linear',
+(TaskRunner pid=56225)                       'tokenizer_path': '/scratch/js15262/.huggingface/hub/models--Qwen--Qwen3-1.7B-Base/snapshots/ea980cb0a6c2ae4b936e82123acc929f1cec04c1',
+(TaskRunner pid=56225)                       'trust_remote_code': False,
+(TaskRunner pid=56225)                       'use_remove_padding': False,
+(TaskRunner pid=56225)                       'use_shm': False},
+(TaskRunner pid=56225)             'optim': {'lr': 1e-05,
+(TaskRunner pid=56225)                       'lr_warmup_steps_ratio': 0.0,
+(TaskRunner pid=56225)                       'min_lr_ratio': None,
+(TaskRunner pid=56225)                       'total_training_steps': -1,
+(TaskRunner pid=56225)                       'warmup_style': 'constant',
+(TaskRunner pid=56225)                       'weight_decay': 0.01},
+(TaskRunner pid=56225)             'ppo_epochs': 1,
+(TaskRunner pid=56225)             'ppo_max_token_len_per_gpu': 32768,
+(TaskRunner pid=56225)             'ppo_micro_batch_size': None,
+(TaskRunner pid=56225)             'ppo_micro_batch_size_per_gpu': None,
+(TaskRunner pid=56225)             'ppo_mini_batch_size': 64,
+(TaskRunner pid=56225)             'profiler': {'all_ranks': False, 'discrete': False, 'ranks': None},
+(TaskRunner pid=56225)             'rollout_n': 8,
+(TaskRunner pid=56225)             'shuffle': False,
+(TaskRunner pid=56225)             'strategy': 'fsdp',
+(TaskRunner pid=56225)             'ulysses_sequence_parallel_size': 1,
+(TaskRunner pid=56225)             'use_dynamic_bsz': False},
+(TaskRunner pid=56225)  'custom_reward_function': {'name': 'compute_score_batch',
+(TaskRunner pid=56225)                             'path': '/scratch/js15262/rl-forgetting-experiments/reward-signal-analysis/reward_function.py'},
+(TaskRunner pid=56225)  'data': {'custom_cls': {'name': None, 'path': None},
+(TaskRunner pid=56225)           'filter_overlong_prompts': True,
+(TaskRunner pid=56225)           'filter_overlong_prompts_workers': 1,
+(TaskRunner pid=56225)           'image_key': 'images',
+(TaskRunner pid=56225)           'max_prompt_length': 1024,
+(TaskRunner pid=56225)           'max_response_length': 3072,
+(TaskRunner pid=56225)           'prompt_key': 'prompt',
+(TaskRunner pid=56225)           'return_full_prompt': False,
+(TaskRunner pid=56225)           'return_raw_chat': False,
+(TaskRunner pid=56225)           'return_raw_input_ids': False,
+(TaskRunner pid=56225)           'reward_fn_key': 'data_source',
+(TaskRunner pid=56225)           'seed': 1,
+(TaskRunner pid=56225)           'shuffle': True,
+(TaskRunner pid=56225)           'tokenizer': None,
+(TaskRunner pid=56225)           'train_batch_size': 64,
+(TaskRunner pid=56225)           'train_files': '/scratch/js15262/rl-forgetting-experiments/datasets/mbpp/train.parquet',
+(TaskRunner pid=56225)           'truncation': 'error',
+(TaskRunner pid=56225)           'trust_remote_code': False,
+(TaskRunner pid=56225)           'use_shm': False,
+(TaskRunner pid=56225)           'val_batch_size': None,
+(TaskRunner pid=56225)           'val_files': ['/scratch/js15262/rl-forgetting-experiments/datasets/mbpp/test.parquet'],
+(TaskRunner pid=56225)           'validation_shuffle': False,
+(TaskRunner pid=56225)           'video_key': 'videos'},
+(TaskRunner pid=56225)  'ray_init': {'num_cpus': None, 'timeline_json_file': None},
+(TaskRunner pid=56225)  'reward_model': {'enable': False,
+(TaskRunner pid=56225)                   'forward_max_token_len_per_gpu': 32768,
+(TaskRunner pid=56225)                   'launch_reward_fn_async': False,
+(TaskRunner pid=56225)                   'max_length': None,
+(TaskRunner pid=56225)                   'micro_batch_size': None,
+(TaskRunner pid=56225)                   'micro_batch_size_per_gpu': None,
+(TaskRunner pid=56225)                   'model': {'external_lib': None,
+(TaskRunner pid=56225)                             'fsdp_config': {'forward_prefetch': False,
+(TaskRunner pid=56225)                                             'fsdp_size': -1,
+(TaskRunner pid=56225)                                             'param_offload': False,
+(TaskRunner pid=56225)                                             'reshard_after_forward': True,
+(TaskRunner pid=56225)                                             'wrap_policy': {'min_num_params': 0}},
+(TaskRunner pid=56225)                             'input_tokenizer': '/scratch/js15262/.huggingface/hub/models--Qwen--Qwen3-1.7B-Base/snapshots/ea980cb0a6c2ae4b936e82123acc929f1cec04c1',
+(TaskRunner pid=56225)                             'path': '~/models/FsfairX-LLaMA3-RM-v0.1',
+(TaskRunner pid=56225)                             'trust_remote_code': False,
+(TaskRunner pid=56225)                             'use_fused_kernels': False,
+(TaskRunner pid=56225)                             'use_remove_padding': False,
+(TaskRunner pid=56225)                             'use_shm': False},
+(TaskRunner pid=56225)                   'profiler': {'all_ranks': False,
+(TaskRunner pid=56225)                                'discrete': False,
+(TaskRunner pid=56225)                                'ranks': None},
+(TaskRunner pid=56225)                   'reward_manager': 'batch',
+(TaskRunner pid=56225)                   'sandbox_fusion': {'max_concurrent': 64,(TaskRunner pid=56225) Setting TOKENIZERS_PARALLELISM=false for forked processes.
+(TaskRunner pid=56225) WARNING:2026-11-01 11:14:54,875:Setting TOKENIZERS_PARALLELISM=false for forked processes.
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) Setting TOKENIZERS_PARALLELISM=false for forked processes.
+(TaskRunner pid=56225) WARNING:2026-11-01 11:14:56,046:Setting TOKENIZERS_PARALLELISM=false for forked processes.
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) DeprecationWarning: `ray.state.available_resources_per_node` is a private attribute and access will be removed in a future Ray version.
+(TaskRunner pid=56225) WARNING:2026-11-01 11:15:02,509:Waiting for register center actor yFzIoP_register_center to be ready. Elapsed time: 0 seconds out of 300 seconds.
+(pid=65344) /scratch/js15262/rl-forgetting-experiments/reward-signal-analysis/verl/__init__.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
+(pid=65344)   import pkg_resources
+(TaskRunner pid=56225) WARNING:2026-11-01 11:15:32,526:Waiting for register center actor yFzIoP_register_center to be ready. Elapsed time: 30 seconds out of 300 seconds.
+(pid=69481) /scratch/js15262/rl-forgetting-experiments/reward-signal-analysis/verl/__init__.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
+(pid=69481)   import pkg_resources
+
+(TaskRunner pid=56225)                                      'memory_limit_mb': 1024,
+(TaskRunner pid=56225)                                      'url': None},
+(TaskRunner pid=56225)                   'strategy': 'fsdp',
+(TaskRunner pid=56225)                   'ulysses_sequence_parallel_size': 1,
+(TaskRunner pid=56225)                   'use_dynamic_bsz': False},
+(TaskRunner pid=56225)  'trainer': {'balance_batch': True,
+(TaskRunner pid=56225)              'buffer_data_dir': '/scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/buffer/replayed',
+(TaskRunner pid=56225)              'controller_nsight_options': {'cuda-graph-trace': 'graph',
+(TaskRunner pid=56225)                                            'cuda-memory-usage': 'true',
+(TaskRunner pid=56225)                                            'trace': 'cuda,nvtx,cublas,ucx'},
+(TaskRunner pid=56225)              'critic_warmup': 0,
+(TaskRunner pid=56225)              'default_hdfs_dir': None,
+(TaskRunner pid=56225)              'default_local_dir': '/scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints',
+(TaskRunner pid=56225)              'del_local_ckpt_after_load': False,
+(TaskRunner pid=56225)              'device': 'cuda',
+(TaskRunner pid=56225)              'esi_redundant_time': 0,
+(TaskRunner pid=56225)              'experiment_name': 'qwen3_1p7b_base_mbpp_nobuf_lr6e7',
+(TaskRunner pid=56225)              'keep_full_ckpt_last_n': 2,
+(TaskRunner pid=56225)              'log_val_generations': 0,
+(TaskRunner pid=56225)              'logger': ['console', 'wandb'],
+(TaskRunner pid=56225)              'max_actor_ckpt_to_keep': None,
+(TaskRunner pid=56225)              'max_critic_ckpt_to_keep': None,
+(TaskRunner pid=56225)              'n_gpus_per_node': 2,
+(TaskRunner pid=56225)              'nnodes': 1,
+(TaskRunner pid=56225)              'profile_steps': None,
+(TaskRunner pid=56225)              'project_name': 'rl_replay_code_mbpp',
+(TaskRunner pid=56225)              'ray_wait_register_center_timeout': 300,
+(TaskRunner pid=56225)              'resume_from_path': None,
+(TaskRunner pid=56225)              'resume_mode': 'auto',
+(TaskRunner pid=56225)              'rollout_data_dir': '/scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/training',
+(TaskRunner pid=56225)              'save_freq': 10,
+(TaskRunner pid=56225)              'test_freq': 50,
+(TaskRunner pid=56225)              'total_epochs': 200,
+(TaskRunner pid=56225)              'total_training_steps': None,
+(TaskRunner pid=56225)              'val_before_train': False,
+(TaskRunner pid=56225)              'val_only': False,
+(TaskRunner pid=56225)              'validation_data_dir': '/scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/validation',
+(TaskRunner pid=56225)              'worker_nsight_options': {'capture-range': 'cudaProfilerApi',
+(TaskRunner pid=56225)                                        'capture-range-end': None,
+(TaskRunner pid=56225)                                        'cuda-graph-trace': 'graph',
+(TaskRunner pid=56225)                                        'cuda-memory-usage': 'true',
+(TaskRunner pid=56225)                                        'kill': 'none',
+(TaskRunner pid=56225)                                        'trace': 'cuda,nvtx,cublas,ucx'}}}
+(TaskRunner pid=56225) using customized reward function 'compute_score_batch' from '/scratch/js15262/rl-forgetting-experiments/reward-signal-analysis/reward_function.py'
+(TaskRunner pid=56225) using customized reward function 'compute_score_batch' from '/scratch/js15262/rl-forgetting-experiments/reward-signal-analysis/reward_function.py'
+(TaskRunner pid=56225) Using dataset class: RLHFDataset
+(TaskRunner pid=56225) dataset len: 320
+(TaskRunner pid=56225) filter dataset len: 320
+(TaskRunner pid=56225) Using dataset class: RLHFDataset
+(TaskRunner pid=56225) dataset len: 378
+(TaskRunner pid=56225) filter dataset len: 378
+(TaskRunner pid=56225) [validate_config] All configuration checks passed successfully!
+(TaskRunner pid=56225) Size of train dataloader: 5, Size of val dataloader: 1
+(TaskRunner pid=56225) Total training steps: 1000
+(TaskRunner pid=56225) colocated worker base class <class 'verl.single_controller.base.worker.Worker'>
+(WorkerDict pid=65344) Model config after override: Qwen3Config {
+(WorkerDict pid=65344)   "architectures": [
+(WorkerDict pid=65344)     "Qwen3ForCausalLM"
+(WorkerDict pid=65344)   ],
+(WorkerDict pid=65344)   "attention_bias": false,
+(WorkerDict pid=65344)   "attention_dropout": 0.0,
+(WorkerDict pid=65344)   "dtype": "bfloat16",
+(WorkerDict pid=65344)   "eos_token_id": 151643,
+(WorkerDict pid=65344)   "head_dim": 128,
+(WorkerDict pid=65344)   "hidden_act": "silu",
+(WorkerDict pid=65344)   "hidden_size": 2048,
+(WorkerDict pid=65344)   "initializer_range": 0.02,
+(WorkerDict pid=65344)   "intermediate_size": 6144,
+(WorkerDict pid=65344)   "layer_types": [
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention",
+(WorkerDict pid=65344)     "full_attention"
+(WorkerDict pid=65344)   ],
+(WorkerDict pid=65344)   "max_position_embeddings": 32768,
+(WorkerDict pid=65344)   "max_window_layers": 28,
+(WorkerDict pid=65344)   "model_type": "qwen3",
+(WorkerDict pid=65344)   "num_attention_heads": 16,
+(WorkerDict pid=65344)   "num_hidden_layers": 28,
+(WorkerDict pid=65344)   "num_key_value_heads": 8,
+(WorkerDict pid=65344)   "pad_token_id": 151643,
+(WorkerDict pid=65344)   "rms_norm_eps": 1e-06,
+(WorkerDict pid=65344)   "rope_scaling": null,
+(WorkerDict pid=65344)   "rope_theta": 1000000,
+(WorkerDict pid=65344)   "sliding_window": null,
+(WorkerDict pid=65344)   "tie_word_embeddings": true,
+(WorkerDict pid=65344)   "transformers_version": "4.57.0",
+(WorkerDict pid=65344)   "use_cache": true,
+(WorkerDict pid=65344)   "use_sliding_window": false,
+(WorkerDict pid=65344)   "vocab_size": 151936
+(WorkerDict pid=65344) }(WorkerDict pid=65344) `torch_dtype` is deprecated! Use `dtype` instead!
+(WorkerDict pid=65344) Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
+(WorkerDict pid=65344) Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`
+(WorkerDict pid=65344) `torch_dtype` is deprecated! Use `dtype` instead! [repeated 2x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)
+(WorkerDict pid=69481) Flash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen3Model is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)` [repeated 2x across cluster]
+(WorkerDict pid=69481) /scratch/js15262/env/reward-signal/lib/python3.11/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+(WorkerDict pid=69481)   warnings.warn(
+(WorkerDict pid=69481) `torch_dtype` is deprecated! Use `dtype` instead!
+(TaskRunner pid=56225) /scratch/js15262/env/reward-signal/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type.
+(TaskRunner pid=56225) /scratch/js15262/env/reward-signal/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type.
+(TaskRunner pid=56225) wandb: Currently logged in as: jingyanshen (jingyanshen-new-york-university) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+(WorkerDict pid=65344) /scratch/js15262/env/reward-signal/lib/python3.11/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+(TaskRunner pid=56225)   warnings.warn( [repeated 3x across cluster]
+(TaskRunner pid=56225) wandb: setting up run osw6fr0g
+(TaskRunner pid=56225) wandb: Tracking run with wandb version 0.22.2
+(TaskRunner pid=56225) wandb: Run data is saved locally in /scratch/js15262/rl-forgetting-experiments/reward-signal-analysis/wandb/run-20261101_111747-osw6fr0g
+(TaskRunner pid=56225) wandb: Run `wandb offline` to turn off syncing.
+(TaskRunner pid=56225) wandb: Syncing run qwen3_1p7b_base_mbpp_nobuf_lr6e7
+(TaskRunner pid=56225) wandb: ⭐️ View project at https://wandb.ai/jingyanshen-new-york-university/rl_replay_code_mbpp
+(TaskRunner pid=56225) wandb: 🚀 View run at https://wandb.ai/jingyanshen-new-york-university/rl_replay_code_mbpp/runs/osw6fr0g
+(TaskRunner pid=56225) wandb: Detected [openai] in use.
+(TaskRunner pid=56225) wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
+(TaskRunner pid=56225) wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
+(WorkerDict pid=65344) INFO:2026-11-01 11:18:00,647:[Rank 0] Loaded model from /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_990/actor/model_world_size_2_rank_0.pt
+(WorkerDict pid=65344) INFO:2026-11-01 11:18:07,693:[Rank 0] Loaded optimizer from /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_990/actor/optim_world_size_2_rank_0.pt
+(WorkerDict pid=65344) INFO:2026-11-01 11:18:07,709:[Rank 0] Loaded rng from /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_990/actor/extra_state_world_size_2_rank_0.pt
+(WorkerDict pid=65344) INFO:2026-11-01 11:18:07,709:[Rank 0] Loaded lr_scheduler from /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_990/actor/extra_state_world_size_2_rank_0.pt
+(WorkerDict pid=69481) INFO:2026-11-01 11:18:00,647:[Rank 1] Loaded model from /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_990/actor/model_world_size_2_rank_1.pt
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) 
+(WorkerDict pid=69481) INFO:2026-11-01 11:18:08,314:[Rank 1] Loaded optimizer from /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_990/actor/optim_world_size_2_rank_1.pt
+(WorkerDict pid=69481) INFO:2026-11-01 11:18:08,328:[Rank 1] Loaded rng from /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_990/actor/extra_state_world_size_2_rank_1.pt
+(WorkerDict pid=69481) INFO:2026-11-01 11:18:08,328:[Rank 1] Loaded lr_scheduler from /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_990/actor/extra_state_world_size_2_rank_1.pt
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpml6oej4x.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpk1knkkjr.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpnuw9eq5h.py']' timed out after 10 seconds"
+
+(WorkerDict pid=65344) 
+(WorkerDict pid=65344) Monkey patch _flash_attention_forward in transformers.integrations.flash_attention
+(WorkerDict pid=65344) Skipping monkey patch for Qwen3ForCausalLM as use_fused_kernels is False or fused_kernels_backend is torch
+(WorkerDict pid=65344) Qwen3ForCausalLM contains 1.72B parameters
+(WorkerDict pid=65344) wrap_policy: functools.partial(<function _or_policy at 0x150f278cbb00>, policies=[functools.partial(<function transformer_auto_wrap_policy at 0x150f278cb9c0>, transformer_layer_cls={<class 'transformers.models.qwen3.modeling_qwen3.Qwen3DecoderLayer'>})])
+(WorkerDict pid=65344) NCCL version 2.21.5+cuda12.4
+(WorkerDict pid=65344) Total steps: 1000, num_warmup_steps: 0
+(WorkerDict pid=65344) Actor use_remove_padding=True
+(WorkerDict pid=65344) Actor use_fused_kernels=False
+(WorkerDict pid=65344) WARNING 11-01 11:17:25 [arg_utils.py:1658] VLLM_ATTENTION_BACKEND=XFORMERS is not supported by the V1 Engine. Falling back to V0. We recommend to remove VLLM_ATTENTION_BACKEND=XFORMERS from your config in favor of the V1 Engine.
+(WorkerDict pid=65344) WARNING 11-01 11:17:25 [cuda.py:93] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used
+(WorkerDict pid=69481) Monkey patch _flash_attention_forward in transformers.integrations.flash_attention
+(WorkerDict pid=69481) Skipping monkey patch for Qwen3ForCausalLM as use_fused_kernels is False or fused_kernels_backend is torch
+(WorkerDict pid=69481) kwargs: {'n': 8, 'logprobs': 0, 'max_tokens': 3072, 'detokenize': False, 'temperature': 1.0, 'top_k': -1, 'top_p': 1, 'ignore_eos': False}
+(WorkerDict pid=69481) WARNING 11-01 11:17:25 [arg_utils.py:1658] VLLM_ATTENTION_BACKEND=XFORMERS is not supported by the V1 Engine. Falling back to V0. We recommend to remove VLLM_ATTENTION_BACKEND=XFORMERS from your config in favor of the V1 Engine.
+(WorkerDict pid=69481) WARNING 11-01 11:17:25 [cuda.py:93] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used
+(TaskRunner pid=56225) Found checkpoint: %s /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_990
+(TaskRunner pid=56225) Load from checkpoint folder: /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_990
+(TaskRunner pid=56225) Setting global step to 990
+(TaskRunner pid=56225) Resuming from /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_990
+(WorkerDict pid=65344) kwargs: {'n': 8, 'logprobs': 0, 'max_tokens': 3072, 'detokenize': False, 'temperature': 1.0, 'top_k': -1, 'top_p': 1, 'ignore_eos': False}
+(TaskRunner pid=56225) [sampler] global_step=991 -> epoch 198 (position 1/5), draw 396
+(TaskRunner pid=56225) [sampler] global_step=991 -> epoch 198 (position 1/5), draw 396
+(TaskRunner pid=56225) tensor([0., 0., 0.,  ..., 0., 0., 0.])
+(TaskRunner pid=56225) Dumped generations to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/training/991.jsonl
+(TaskRunner pid=56225) step:991 - global_seqlen/min:84452.000 - global_seqlen/max:84591.000 - global_seqlen/minmax_diff:139.000 - global_seqlen/balanced_min:84521.000 - global_seqlen/balanced_max:84522.000 - global_seqlen/mean:84521.500 - actor/entropy:0.060 - actor/pg_loss:0.004 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/pg_clipfrac_lower:0.000 - actor/grad_norm:0.062 - perf/mfu/actor:0.039 - perf/max_memory_allocated_gb:114.090 - perf/max_memory_reserved_gb:124.375 - perf/cpu_memory_used_gb:119.565 - actor/lr:0.000 - training/global_step:991.000 - training/epoch:1.000 - critic/score/mean:0.949 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.949 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:-0.000 - critic/advantages/max:0.250 - critic/advantages/min:-0.750 - critic/returns/mean:-0.000 - critic/returns/max:0.250 - critic/returns/min:-0.750 - response_length/mean:177.771 - response_length/max:478.000 - response_length/min:92.000 - response_length/clip_ratio:0.000 - prompt_length/mean:152.391 - prompt_length/max:204.000 - prompt_length/min:128.000 - prompt_length/clip_ratio:0.000 - timing_s/generate_sequences:16.549 - timing_s/reshard:0.493 - timing_s/gen:19.796 - timing_s/reward:40.580 - timing_s/old_log_prob:27.405 - timing_s/adv:0.020 - timing_s/update_actor:27.180 - timing_s/dump_rollout_generations:0.238 - timing_s/step:115.289 - timing_per_token_ms/gen:0.217 - timing_per_token_ms/update_actor:0.161 - timing_per_token_ms/adv:0.000 - perf/total_num_tokens:169043.000 - perf/time_per_step:115.289 - perf/throughput:733.130
+(TaskRunner pid=56225) tensor([0., 0., 0.,  ..., 0., 0., 0.])
+(TaskRunner pid=56225) Dumped generations to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/training/992.jsonl
+(TaskRunner pid=56225) step:992 - global_seqlen/min:82691.000 - global_seqlen/max:90520.000 - global_seqlen/minmax_diff:7829.000 - global_seqlen/balanced_min:86605.000 - global_seqlen/balanced_max:86606.000 - global_seqlen/mean:86605.500 - actor/entropy:0.060 - actor/pg_loss:0.001 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/pg_clipfrac_lower:0.000 - actor/grad_norm:0.104 - perf/mfu/actor:0.042 - perf/max_memory_allocated_gb:114.662 - perf/max_memory_reserved_gb:124.506 - perf/cpu_memory_used_gb:121.139 - actor/lr:0.000 - training/global_step:992.000 - training/epoch:1.000 - critic/score/mean:0.881 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.881 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:0.000 - critic/advantages/max:0.125 - critic/advantages/min:-0.875 - critic/returns/mean:0.000 - critic/returns/max:0.125 - critic/returns/min:-0.875 - response_length/mean:184.709 - response_length/max:525.000 - response_length/min:86.000 - response_length/clip_ratio:0.000 - prompt_length/mean:153.594 - prompt_length/max:234.000 - prompt_length/min:125.000 - prompt_length/clip_ratio:0.000 - timing_s/generate_sequences:17.488 - timing_s/reshard:0.581 - timing_s/gen:20.041 - timing_s/reward:40.862 - timing_s/old_log_prob:10.264 - timing_s/adv:0.020 - timing_s/update_actor:26.092 - timing_s/dump_rollout_generations:0.233 - timing_s/step:97.578 - timing_per_token_ms/gen:0.212 - timing_per_token_ms/update_actor:0.151 - timing_per_token_ms/adv:0.000 - perf/total_num_tokens:173211.000 - perf/time_per_step:97.578 - perf/throughput:887.553
+(TaskRunner pid=56225) tensor([0., 0., 0.,  ..., 0., 0., 0.])
+(TaskRunner pid=56225) Dumped generations to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/training/993.jsonl
+(TaskRunner pid=56225) step:993 - global_seqlen/min:89405.000 - global_seqlen/max:89553.000 - global_seqlen/minmax_diff:148.000 - global_seqlen/balanced_min:89479.000 - global_seqlen/balanced_max:89479.000 - global_seqlen/mean:89479.000 - actor/entropy:0.065 - actor/pg_loss:-0.003 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/pg_clipfrac_lower:0.000 - actor/grad_norm:0.096 - perf/mfu/actor:0.044 - perf/max_memory_allocated_gb:115.934 - perf/max_memory_reserved_gb:124.506 - perf/cpu_memory_used_gb:116.739 - actor/lr:0.000 - training/global_step:993.000 - training/epoch:1.000 - critic/score/mean:0.822 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.822 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:0.002 - critic/advantages/max:0.750 - critic/advantages/min:-0.875 - critic/returns/mean:0.002 - critic/returns/max:0.750 - critic/returns/min:-0.875 - response_length/mean:195.059 - response_length/max:672.000 - response_length/min:74.000 - response_length/clip_ratio:0.000 - prompt_length/mean:154.469 - prompt_length/max:336.000 - prompt_length/min:123.000 - prompt_length/clip_ratio:0.000 - timing_s/generate_sequences:18.792 - timing_s/reshard:0.318 - timing_s/gen:23.185 - timing_s/reward:58.401 - timing_s/old_log_prob:8.458 - timing_s/adv:0.019 - timing_s/update_actor:25.653 - timing_s/dump_rollout_generations:0.237 - timing_s/step:116.006 - timing_per_token_ms/gen:0.232 - timing_per_token_ms/update_actor:0.143 - timing_per_token_ms/adv:0.000 - perf/total_num_tokens:178958.000 - perf/time_per_step:116.006 - perf/throughput:771.329(TaskRunner pid=56225) 
+(TaskRunner pid=56225) Traceback (most recent call last):
+(TaskRunner pid=56225)   File "/scratch/js15262/rl-forgetting-experiments/reward-signal-analysis/verl/utils/reward_score/livecodebench/compute_score.py", line 368, in compute_score
+(TaskRunner pid=56225)     tree = ast.parse(solution)
+(TaskRunner pid=56225)            ^^^^^^^^^^^^^^^^^^^
+(TaskRunner pid=56225)   File "/scratch/js15262/env/reward-signal/lib/python3.11/ast.py", line 50, in parse
+(TaskRunner pid=56225)     return compile(source, filename, mode, flags,
+(TaskRunner pid=56225)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(TaskRunner pid=56225)   File "<unknown>", line 8
+(TaskRunner pid=56225)     exclude = new exclude
+(TaskRunner pid=56225)                   ^^^^^^^
+(TaskRunner pid=56225) SyntaxError: invalid syntax
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpvm2yvvci.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpogdp_pgr.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmp7e2bq3q6.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpfjj04dch.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpf8ynf9y6.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpihgidw4n.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmp9h5yegap.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmp7il_9j05.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpevzhzb7e.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpizol0y8b.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpkgtmlwlz.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpw4bpqe0p.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) Traceback (most recent call last):
+(TaskRunner pid=56225)   File "/scratch/js15262/rl-forgetting-experiments/reward-signal-analysis/verl/utils/reward_score/livecodebench/compute_score.py", line 368, in compute_score
+(TaskRunner pid=56225)     tree = ast.parse(solution)
+(TaskRunner pid=56225)            ^^^^^^^^^^^^^^^^^^^
+(TaskRunner pid=56225)   File "/scratch/js15262/env/reward-signal/lib/python3.11/ast.py", line 50, in parse
+(TaskRunner pid=56225)     return compile(source, filename, mode, flags,
+(TaskRunner pid=56225)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(TaskRunner pid=56225)   File "<unknown>", line 7
+(TaskRunner pid=56225)     return 'Not matched!
+(TaskRunner pid=56225)            ^
+(TaskRunner pid=56225) SyntaxError: unterminated string literal (detected at line 7)
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) Traceback (most recent call last):
+(TaskRunner pid=56225)   File "/scratch/js15262/rl-forgetting-experiments/reward-signal-analysis/verl/utils/reward_score/livecodebench/compute_score.py", line 368, in compute_score
+(TaskRunner pid=56225)     tree = ast.parse(solution)
+(TaskRunner pid=56225)            ^^^^^^^^^^^^^^^^^^^
+(TaskRunner pid=56225)   File "/scratch/js15262/env/reward-signal/lib/python3.11/ast.py", line 50, in parse
+(TaskRunner pid=56225)     return compile(source, filename, mode, flags,
+(TaskRunner pid=56225)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(TaskRunner pid=56225)   File "<unknown>", line 1
+(TaskRunner pid=56225)     import math
+(TaskRunner pid=56225) IndentationError: unexpected indent
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpozz_7svl.py']' timed out after 10 seconds"
+
+(TaskRunner pid=56225) tensor([0., 0., 0.,  ..., 0., 0., 0.])
+(TaskRunner pid=56225) Dumped generations to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/training/994.jsonl
+(TaskRunner pid=56225) step:994 - global_seqlen/min:88499.000 - global_seqlen/max:90517.000 - global_seqlen/minmax_diff:2018.000 - global_seqlen/balanced_min:89508.000 - global_seqlen/balanced_max:89508.000 - global_seqlen/mean:89508.000 - actor/entropy:0.057 - actor/pg_loss:-0.003 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/pg_clipfrac_lower:0.000 - actor/grad_norm:0.093 - perf/mfu/actor:0.049 - perf/max_memory_allocated_gb:115.934 - perf/max_memory_reserved_gb:124.506 - perf/cpu_memory_used_gb:118.702 - actor/lr:0.000 - training/global_step:994.000 - training/epoch:1.000 - critic/score/mean:0.869 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.869 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:-0.000 - critic/advantages/max:0.125 - critic/advantages/min:-0.875 - critic/returns/mean:-0.000 - critic/returns/max:0.125 - critic/returns/min:-0.875 - response_length/mean:191.703 - response_length/max:632.000 - response_length/min:93.000 - response_length/clip_ratio:0.000 - prompt_length/mean:157.938 - prompt_length/max:267.000 - prompt_length/min:129.000 - prompt_length/clip_ratio:0.000 - timing_s/generate_sequences:15.722 - timing_s/reshard:0.878 - timing_s/gen:19.217 - timing_s/reward:45.416 - timing_s/old_log_prob:9.948 - timing_s/adv:0.025 - timing_s/update_actor:23.361 - timing_s/dump_rollout_generations:0.248 - timing_s/step:98.252 - timing_per_token_ms/gen:0.196 - timing_per_token_ms/update_actor:0.130 - timing_per_token_ms/adv:0.000 - perf/total_num_tokens:179016.000 - perf/time_per_step:98.252 - perf/throughput:911.007
+(TaskRunner pid=56225) tensor([0., 0., 0.,  ..., 0., 0., 0.])
+(TaskRunner pid=56225) Dumped generations to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/training/995.jsonl
+(TaskRunner pid=56225) step:995 - global_seqlen/min:84736.000 - global_seqlen/max:84865.000 - global_seqlen/minmax_diff:129.000 - global_seqlen/balanced_min:84800.000 - global_seqlen/balanced_max:84801.000 - global_seqlen/mean:84800.500 - actor/entropy:0.058 - actor/pg_loss:-0.007 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/pg_clipfrac_lower:0.000 - actor/grad_norm:0.119 - perf/mfu/actor:0.041 - perf/max_memory_allocated_gb:115.934 - perf/max_memory_reserved_gb:124.506 - perf/cpu_memory_used_gb:121.864 - actor/lr:0.000 - training/global_step:995.000 - training/epoch:1.000 - critic/score/mean:0.846 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.846 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:0.000 - critic/advantages/max:0.125 - critic/advantages/min:-0.875 - critic/returns/mean:0.000 - critic/returns/max:0.125 - critic/returns/min:-0.875 - response_length/mean:185.174 - response_length/max:496.000 - response_length/min:77.000 - response_length/clip_ratio:0.000 - prompt_length/mean:146.078 - prompt_length/max:198.000 - prompt_length/min:124.000 - prompt_length/clip_ratio:0.000 - timing_s/generate_sequences:15.478 - timing_s/reshard:0.313 - timing_s/gen:17.095 - timing_s/reward:38.167 - timing_s/old_log_prob:6.894 - timing_s/adv:0.019 - timing_s/update_actor:26.368 - timing_s/dump_rollout_generations:0.238 - timing_s/step:88.845 - timing_per_token_ms/gen:0.180 - timing_per_token_ms/update_actor:0.155 - timing_per_token_ms/adv:0.000 - perf/total_num_tokens:169601.000 - perf/time_per_step:88.845 - perf/throughput:954.476
+(TaskRunner pid=56225) [sampler] global_step=996 -> epoch 199 (position 1/5), draw 398
+(TaskRunner pid=56225) tensor([0., 0., 0.,  ..., 0., 0., 0.])
+(TaskRunner pid=56225) Dumped generations to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/training/996.jsonl
+(TaskRunner pid=56225) step:996 - global_seqlen/min:84937.000 - global_seqlen/max:85613.000 - global_seqlen/minmax_diff:676.000 - global_seqlen/balanced_min:85275.000 - global_seqlen/balanced_max:85275.000 - global_seqlen/mean:85275.000 - actor/entropy:0.066 - actor/pg_loss:-0.001 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/pg_clipfrac_lower:0.000 - actor/grad_norm:0.142 - perf/mfu/actor:0.043 - perf/max_memory_allocated_gb:115.934 - perf/max_memory_reserved_gb:124.506 - perf/cpu_memory_used_gb:122.688 - actor/lr:0.000 - training/global_step:996.000 - training/epoch:2.000 - critic/score/mean:0.881 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.881 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:0.000 - critic/advantages/max:0.125 - critic/advantages/min:-0.875 - critic/returns/mean:0.000 - critic/returns/max:0.125 - critic/returns/min:-0.875 - response_length/mean:178.762 - response_length/max:436.000 - response_length/min:68.000 - response_length/clip_ratio:0.000 - prompt_length/mean:154.344 - prompt_length/max:238.000 - prompt_length/min:124.000 - prompt_length/clip_ratio:0.000 - timing_s/generate_sequences:16.447 - timing_s/reshard:0.327 - timing_s/gen:20.275 - timing_s/reward:37.629 - timing_s/old_log_prob:7.846 - timing_s/adv:0.028 - timing_s/update_actor:25.378 - timing_s/dump_rollout_generations:0.241 - timing_s/step:91.483 - timing_per_token_ms/gen:0.222 - timing_per_token_ms/update_actor:0.149 - timing_per_token_ms/adv:0.000 - perf/total_num_tokens:170550.000 - perf/time_per_step:91.483 - perf/throughput:932.137
+(TaskRunner pid=56225) tensor([0., 0., 0.,  ..., 0., 0., 0.])
+(TaskRunner pid=56225) Dumped generations to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/training/997.jsonl
+(TaskRunner pid=56225) step:997 - global_seqlen/min:83348.000 - global_seqlen/max:87012.000 - global_seqlen/minmax_diff:3664.000 - global_seqlen/balanced_min:85180.000 - global_seqlen/balanced_max:85180.000 - global_seqlen/mean:85180.000 - actor/entropy:0.058 - actor/pg_loss:-0.003 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/pg_clipfrac_lower:0.000 - actor/grad_norm:0.166 - perf/mfu/actor:0.043 - perf/max_memory_allocated_gb:115.934 - perf/max_memory_reserved_gb:124.506 - perf/cpu_memory_used_gb:123.358 - actor/lr:0.000 - training/global_step:997.000 - training/epoch:2.000 - critic/score/mean:0.910 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.910 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:-0.000 - critic/advantages/max:0.125 - critic/advantages/min:-0.875 - critic/returns/mean:-0.000 - critic/returns/max:0.125 - critic/returns/min:-0.875 - response_length/mean:180.969 - response_length/max:536.000 - response_length/min:89.000 - response_length/clip_ratio:0.000 - prompt_length/mean:151.766 - prompt_length/max:205.000 - prompt_length/min:125.000 - prompt_length/clip_ratio:0.000 - timing_s/generate_sequences:16.853 - timing_s/reshard:0.474 - timing_s/gen:18.450 - timing_s/reward:36.558 - timing_s/old_log_prob:9.104 - timing_s/adv:0.024 - timing_s/update_actor:25.080 - timing_s/dump_rollout_generations:0.274 - timing_s/step:89.550 - timing_per_token_ms/gen:0.199 - timing_per_token_ms/update_actor:0.147 - timing_per_token_ms/adv:0.000 - perf/total_num_tokens:170360.000 - perf/time_per_step:89.550 - perf/throughput:951.201
+(TaskRunner pid=56225) tensor([0., 0., 0.,  ..., 0., 0., 0.])
+(TaskRunner pid=56225) Dumped generations to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/training/998.jsonl
+(TaskRunner pid=56225) step:998 - global_seqlen/min:81833.000 - global_seqlen/max:90001.000 - global_seqlen/minmax_diff:8168.000 - global_seqlen/balanced_min:85917.000 - global_seqlen/balanced_max:85917.000 - global_seqlen/mean:85917.000 - actor/entropy:0.059 - actor/pg_loss:-0.005 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/pg_clipfrac_lower:0.000 - actor/grad_norm:0.097 - perf/mfu/actor:0.042 - perf/max_memory_allocated_gb:115.934 - perf/max_memory_reserved_gb:125.186 - perf/cpu_memory_used_gb:121.610 - actor/lr:0.000 - training/global_step:998.000 - training/epoch:2.000 - critic/score/mean:0.842 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.842 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:0.003 - critic/advantages/max:0.750 - critic/advantages/min:-0.875 - critic/returns/mean:0.003 - critic/returns/max:0.750 - critic/returns/min:-0.875 - response_length/mean:184.551 - response_length/max:705.000 - response_length/min:72.000 - response_length/clip_ratio:0.000 - prompt_length/mean:151.062 - prompt_length/max:336.000 - prompt_length/min:129.000 - prompt_length/clip_ratio:0.000 - timing_s/generate_sequences:17.548 - timing_s/reshard:0.633 - timing_s/gen:22.566 - timing_s/reward:40.129 - timing_s/old_log_prob:7.812 - timing_s/adv:0.019 - timing_s/update_actor:25.830 - timing_s/dump_rollout_generations:0.250 - timing_s/step:96.684 - timing_per_token_ms/gen:0.239 - timing_per_token_ms/update_actor:0.150 - timing_per_token_ms/adv:0.000 - perf/total_num_tokens:171834.000 - perf/time_per_step:96.684 - perf/throughput:888.637(TaskRunner pid=56225) 
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpavbh0hc_.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmp5b6rt6nl.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpprjcb4b4.py']' timed out after 10 seconds"
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case subprocess: the test process did not decide (policy=lenient). First stderr: 'Traceback (most recent call last):\n  File "/state/partition1/job-21591505/tmpmptpo0dd.py", line 42, in <module>\n    assert largest_subset([ 1, 3, 6, 13, 17, 18 ]) == 4\n           ^^^^^^^^^^^^^^^^^^^^^'
+(TaskRunner pid=56225) [compute_score] EXEC_FAILURE at assert_case exception: the test process did not decide (policy=lenient). First stderr: "Command '['python', '/state/partition1/job-21591505/tmpjeh7vh4j.py']' timed out after 10 seconds"
+(WorkerDict pid=69481) INFO:2026-11-01 11:35:37,469:[Rank 1] Saved model to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_1000/actor/model_world_size_2_rank_1.pt
+(WorkerDict pid=65344) INFO:2026-11-01 11:35:48,108:[Rank 0] Saved optim to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_1000/actor/optim_world_size_2_rank_0.pt
+(WorkerDict pid=65344) INFO:2026-11-01 11:35:48,115:[Rank 0] Saved extra_state to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_1000/actor/extra_state_world_size_2_rank_0.pt
+(WorkerDict pid=65344) INFO:2026-11-01 11:35:37,533:[Rank 0] Saved model to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_1000/actor/model_world_size_2_rank_0.pt
+(WorkerDict pid=65344) INFO:2026-11-01 11:35:48,392:[Rank 0] Saved model config and tokenizer class to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_1000/actor/huggingface
+
+(TaskRunner pid=56225) tensor([0., 0., 0.,  ..., 0., 0., 0.])
+(TaskRunner pid=56225) Dumped generations to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/training/999.jsonl
+(TaskRunner pid=56225) step:999 - global_seqlen/min:88909.000 - global_seqlen/max:99113.000 - global_seqlen/minmax_diff:10204.000 - global_seqlen/balanced_min:94011.000 - global_seqlen/balanced_max:94011.000 - global_seqlen/mean:94011.000 - actor/entropy:0.057 - actor/pg_loss:-0.002 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/pg_clipfrac_lower:0.000 - actor/grad_norm:0.148 - perf/mfu/actor:0.047 - perf/max_memory_allocated_gb:115.934 - perf/max_memory_reserved_gb:125.186 - perf/cpu_memory_used_gb:120.883 - actor/lr:0.000 - training/global_step:999.000 - training/epoch:2.000 - critic/score/mean:0.861 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.861 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:-0.001 - critic/advantages/max:0.250 - critic/advantages/min:-0.875 - critic/returns/mean:-0.001 - critic/returns/max:0.250 - critic/returns/min:-0.875 - response_length/mean:212.105 - response_length/max:563.000 - response_length/min:96.000 - response_length/clip_ratio:0.000 - prompt_length/mean:155.125 - prompt_length/max:210.000 - prompt_length/min:123.000 - prompt_length/clip_ratio:0.000 - timing_s/generate_sequences:18.587 - timing_s/reshard:0.485 - timing_s/gen:20.776 - timing_s/reward:40.910 - timing_s/old_log_prob:13.133 - timing_s/adv:0.025 - timing_s/update_actor:25.476 - timing_s/dump_rollout_generations:0.242 - timing_s/step:100.597 - timing_per_token_ms/gen:0.191 - timing_per_token_ms/update_actor:0.135 - timing_per_token_ms/adv:0.000 - perf/total_num_tokens:188022.000 - perf/time_per_step:100.597 - perf/throughput:934.533
+(TaskRunner pid=56225) tensor([0., 0., 0.,  ..., 0., 0., 0.])
+(TaskRunner pid=56225) Dumped generations to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/training/1000.jsonl
+(TaskRunner pid=56225) test_gen_batch meta info: {'eos_token_id': 151643, 'pad_token_id': 151643, 'recompute_log_prob': False, 'do_sample': False, 'validate': True}
+(TaskRunner pid=56225) validation generation end
+(TaskRunner pid=56225) [prompt] system
+(TaskRunner pid=56225) Please reason step by step inside a <think> </think> block, then put your final solution in a single ```python code block.
+(TaskRunner pid=56225) user
+(TaskRunner pid=56225) Write a python function to find the first repeated character in a given string.
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) Your implementation should pass tests like the following (more tests will be used for evaluation):
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) ```python
+(TaskRunner pid=56225) assert first_repeated_char("abcabc") == "a"
+(TaskRunner pid=56225) ```
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) First reason step by step inside <think> and </think>. Then write your complete solution in a single markdown fenced code block using the language tag ```python (open with ```python on its own line, close with ```).
+(TaskRunner pid=56225) assistant
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) [response] To find the first repeated character in a string, we can follow these steps:
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) 1. **Iterate through the string**: We'll go through each character in the string.
+(TaskRunner pid=56225) 2. **Keep track of characters**: We'll use a set to keep track of characters we've already seen.
+(TaskRunner pid=56225) 3. **Check for repetition**: For each character, we'll check if it's already in the set.
+(TaskRunner pid=56225) 4. **Return the first repeated character**: If we find a character that is repeated, we'll return it.
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) Here's the implementation:
+(TaskRunner pid=56225) 
+(TaskRunner pid=56225) ```python
+(TaskRunner pid=56225) def first_repeated_char(s):
+(TaskRunner pid=56225)     seen = set()
+(TaskRunner pid=56225)     for char in s:
+(TaskRunner pid=56225)         if char in seen:
+(TaskRunner pid=56225)             return char
+(TaskRunner pid=56225)         seen.add(char)
+(TaskRunner pid=56225)     return None
+(TaskRunner pid=56225) ```
+(TaskRunner pid=56225) [ground_truth] {"assert_case": ["assert first_repeated_char(\"abcabc\") == \"a\"", "assert first_repeated_char(\"abc\") == None", "assert first_repeated_char(\"123123\") == \"1\""]}
+(TaskRunner pid=56225) [score] {'score': 1.0, 'ground_truth': '{"assert_case": ["assert first_repeated_char(\\"abcabc\\") == \\"a\\"", "assert first_repeated_char(\\"abc\\") == None", "assert first_repeated_char(\\"123123\\") == \\"1\\""]}', 'reward_method': 'CODE', 'has_code_fence': 1.0}
+(TaskRunner pid=56225) Dumped generations to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/rollouts/validation/1000.jsonl
+(TaskRunner pid=56225) local_global_step_folder: /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_1000
+(TaskRunner pid=56225) [ckpt] Converting actor at step 970 to HF format -> /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_970/actor/huggingface
+(TaskRunner pid=56225) [ckpt] Converting actor at step 980 to HF format -> /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_980/actor/huggingface
+(TaskRunner pid=56225) step:1000 - global_seqlen/min:82048.000 - global_seqlen/max:88656.000 - global_seqlen/minmax_diff:6608.000 - global_seqlen/balanced_min:85352.000 - global_seqlen/balanced_max:85352.000 - global_seqlen/mean:85352.000 - actor/entropy:0.060 - actor/pg_loss:-0.002 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/pg_clipfrac_lower:0.000 - actor/grad_norm:0.085 - perf/mfu/actor:0.044 - perf/max_memory_allocated_gb:115.934 - perf/max_memory_reserved_gb:125.234 - perf/cpu_memory_used_gb:121.785 - actor/lr:0.000 - val-core/mbpp/reward/mean@1:0.757 - val-aux/mbpp/outcome_score/mean@1:0.757 - val-aux/mbpp/has_code_fence/mean@1:1.000 - val-aux/mbpp/truncated/mean@1:0.000 - training/global_step:1000.000 - training/epoch:2.000 - critic/score/mean:0.865 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.865 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:0.001 - critic/advantages/max:0.250 - critic/advantages/min:-0.875 - critic/returns/mean:0.001 - critic/returns/max:0.250 - critic/returns/min:-0.875 - response_length/mean:181.234 - response_length/max:403.000 - response_length/min:84.000 - response_length/clip_ratio:0.000 - prompt_length/mean:152.172 - prompt_length/max:267.000 - prompt_length/min:126.000 - prompt_length/clip_ratio:0.000 - timing_s/generate_sequences:12.590 - timing_s/reshard:0.633 - timing_s/gen:15.987 - timing_s/reward:37.642 - timing_s/old_log_prob:12.775 - timing_s/adv:0.025 - timing_s/update_actor:24.686 - timing_s/dump_rollout_generations:0.238 - timing_s/testing:54.594 - timing_s/save_checkpoint:144.778 - timing_s/step:290.772 - timing_per_token_ms/gen:0.172 - timing_per_token_ms/update_actor:0.145 - timing_per_token_ms/adv:0.000 - perf/total_num_tokens:170704.000 - perf/time_per_step:290.772 - perf/throughput:293.536
+(TaskRunner pid=56225) ("Final validation metrics: {'val-core/mbpp/reward/mean@1': "
+(TaskRunner pid=56225)  "np.float64(0.7566137566137566), 'val-aux/mbpp/outcome_score/mean@1': "
+(TaskRunner pid=56225)  "np.float64(0.7566137566137566), 'val-aux/mbpp/has_code_fence/mean@1': "
+(TaskRunner pid=56225)  "np.float64(1.0), 'val-aux/mbpp/truncated/mean@1': np.float64(0.0)}")(TaskRunner pid=56225) 
+(WorkerDict pid=69481) INFO:2026-11-01 11:35:49,107:[Rank 1] Saved optim to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_1000/actor/optim_world_size_2_rank_1.pt
+(WorkerDict pid=69481) INFO:2026-11-01 11:35:49,114:[Rank 1] Saved extra_state to /scratch/js15262/rl-forgetting-experiments/results/code/qwen3_1p7b_base_mbpp_nobuf_lr6e7/checkpoints/global_step_1000/actor/extra_state_world_size_2_rank_1.pt
+(TaskRunner pid=56225) wandb: updating run metadata
+(TaskRunner pid=56225) wandb: 
+(TaskRunner pid=56225) wandb: Run history:
+(TaskRunner pid=56225) wandb:           actor/entropy ▃▃█▁▂█▂▃▁▃
+(TaskRunner pid=56225) wandb:         actor/grad_norm ▁▄▃▃▅▆█▃▇▃
+(TaskRunner pid=56225) wandb:                actor/lr ▁▁▁▁▁▁▁▁▁▁
+(TaskRunner pid=56225) wandb:       actor/pg_clipfrac ▁▁▁▁▁▁▁▁▁▁
+(TaskRunner pid=56225) wandb: actor/pg_clipfrac_lower ▁▁▁▁▁▁▁▁▁▁
+(TaskRunner pid=56225) wandb:           actor/pg_loss █▆▄▄▁▅▄▃▄▄
+(TaskRunner pid=56225) wandb:            actor/ppo_kl ▁▁▁▁▁▁▁▁▁▁
+(TaskRunner pid=56225) wandb:   critic/advantages/max ▂▁█▁▁▁▁█▂▂
+(TaskRunner pid=56225) wandb:  critic/advantages/mean ▂▂▇▂▂▃▂█▁▅
+(TaskRunner pid=56225) wandb:   critic/advantages/min █▁▁▁▁▁▁▁▁▁
+(TaskRunner pid=56225) wandb:                     +50 ...
+(TaskRunner pid=56225) wandb: 
+(TaskRunner pid=56225) wandb: Run summary:
+(TaskRunner pid=56225) wandb:           actor/entropy 0.06008
+(TaskRunner pid=56225) wandb:         actor/grad_norm 0.08475
+(TaskRunner pid=56225) wandb:                actor/lr 0.0
+(TaskRunner pid=56225) wandb:       actor/pg_clipfrac 0
+(TaskRunner pid=56225) wandb: actor/pg_clipfrac_lower 0
+(TaskRunner pid=56225) wandb:           actor/pg_loss -0.00241
+(TaskRunner pid=56225) wandb:            actor/ppo_kl 0
+(TaskRunner pid=56225) wandb:   critic/advantages/max 0.25
+(TaskRunner pid=56225) wandb:  critic/advantages/mean 0.0014
+(TaskRunner pid=56225) wandb:   critic/advantages/min -0.875
+(TaskRunner pid=56225) wandb:                     +50 ...
+(TaskRunner pid=56225) wandb: 
+(TaskRunner pid=56225) wandb: 🚀 View run qwen3_1p7b_base_mbpp_nobuf_lr6e7 at: https://wandb.ai/jingyanshen-new-york-university/rl_replay_code_mbpp/runs/osw6fr0g
+(TaskRunner pid=56225) wandb: ⭐️ View project at: https://wandb.ai/jingyanshen-new-york-university/rl_replay_code_mbpp
+(TaskRunner pid=56225) wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+(TaskRunner pid=56225) wandb: Find logs at: ./wandb/run-20261101_111747-osw6fr0g/logs
+