diff --git a/halfcheetah_medium_expert_sparse/diffusion_123/scheduler/scheduler_config.json b/halfcheetah_medium_expert_sparse/diffusion_123/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0dbe23fd6a0f8f13f0bce60db5ef70d55bec9257 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/diffusion_123/scheduler/scheduler_config.json @@ -0,0 +1,19 @@ +{ + "_class_name": "DDIMScheduler", + "_diffusers_version": "0.36.0", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "clip_sample": true, + "clip_sample_range": 1.0, + "dynamic_thresholding_ratio": 0.995, + "num_train_timesteps": 50000, + "prediction_type": "epsilon", + "rescale_betas_zero_snr": false, + "sample_max_value": 1.0, + "set_alpha_to_one": true, + "steps_offset": 0, + "thresholding": false, + "timestep_spacing": "leading", + "trained_betas": null +} diff --git a/halfcheetah_medium_expert_sparse/diffusion_123/used_config.yaml b/halfcheetah_medium_expert_sparse/diffusion_123/used_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ba1e3dc19b914c481cb5f380a6b33844364d2e4 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/diffusion_123/used_config.yaml @@ -0,0 +1,21 @@ +npz: /public/d4rl/sparse_datasets/diffusion_processed/halfcheetah_medium_expert_sparse_72.5_train.npz +out: /public/gormpo/models/halfcheetah_medium_expert_sparse_3/diffusion +epochs: 100 +batch: 512 +lr: 0.0002 +wd: 0.0 +timesteps: 50000 +checkpoint_every_epochs: 10 +seed: 0 +device: cuda +model: mlp +hidden: 512 +time_emb: 128 +layers: 3 +dropout: 0.0 +wandb: false +wandb_project: GORMPO +wandb_entity: jhhuang_chloe +wandb_run: unconditional_ddim_mlp_halfcheetah_medium_expert_sparse_72.5 +log_every: 100 +samples_every: 1 diff --git a/halfcheetah_medium_expert_sparse/diffusion_456/scheduler/scheduler_config.json b/halfcheetah_medium_expert_sparse/diffusion_456/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0dbe23fd6a0f8f13f0bce60db5ef70d55bec9257 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/diffusion_456/scheduler/scheduler_config.json @@ -0,0 +1,19 @@ +{ + "_class_name": "DDIMScheduler", + "_diffusers_version": "0.36.0", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "clip_sample": true, + "clip_sample_range": 1.0, + "dynamic_thresholding_ratio": 0.995, + "num_train_timesteps": 50000, + "prediction_type": "epsilon", + "rescale_betas_zero_snr": false, + "sample_max_value": 1.0, + "set_alpha_to_one": true, + "steps_offset": 0, + "thresholding": false, + "timestep_spacing": "leading", + "trained_betas": null +} diff --git a/halfcheetah_medium_expert_sparse/diffusion_456/used_config.yaml b/halfcheetah_medium_expert_sparse/diffusion_456/used_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ba1e3dc19b914c481cb5f380a6b33844364d2e4 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/diffusion_456/used_config.yaml @@ -0,0 +1,21 @@ +npz: /public/d4rl/sparse_datasets/diffusion_processed/halfcheetah_medium_expert_sparse_72.5_train.npz +out: /public/gormpo/models/halfcheetah_medium_expert_sparse_3/diffusion +epochs: 100 +batch: 512 +lr: 0.0002 +wd: 0.0 +timesteps: 50000 +checkpoint_every_epochs: 10 +seed: 0 +device: cuda +model: mlp +hidden: 512 +time_emb: 128 +layers: 3 +dropout: 0.0 +wandb: false +wandb_project: GORMPO +wandb_entity: jhhuang_chloe +wandb_run: unconditional_ddim_mlp_halfcheetah_medium_expert_sparse_72.5 +log_every: 100 +samples_every: 1 diff --git a/halfcheetah_medium_expert_sparse/mbpo_based/diffusion/seed_123_0329_180836_halfcheetah_medium_expert_v2_gormpo_sparse_72.5/policy_halfcheetah-medium-expert-v2.pth b/halfcheetah_medium_expert_sparse/mbpo_based/diffusion/seed_123_0329_180836_halfcheetah_medium_expert_v2_gormpo_sparse_72.5/policy_halfcheetah-medium-expert-v2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/halfcheetah_medium_expert_sparse/mbpo_based/kde/seed_123_0329_154353_halfcheetah_medium_expert_v2_gormpo_sparse_72.5/policy_halfcheetah-medium-expert-v2.pth b/halfcheetah_medium_expert_sparse/mbpo_based/kde/seed_123_0329_154353_halfcheetah_medium_expert_v2_gormpo_sparse_72.5/policy_halfcheetah-medium-expert-v2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/halfcheetah_medium_expert_sparse/mbpo_based/neuralODE/seed_123_0329_223759_halfcheetah_medium_expert_v2_gormpo_sparse_72.5/policy_halfcheetah-medium-expert-v2.pth b/halfcheetah_medium_expert_sparse/mbpo_based/neuralODE/seed_123_0329_223759_halfcheetah_medium_expert_v2_gormpo_sparse_72.5/policy_halfcheetah-medium-expert-v2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/halfcheetah_medium_expert_sparse/mbpo_based/realnvp/seed_456_0329_235735_halfcheetah_medium_expert_v2_gormpo_sparse_72.5/policy_halfcheetah-medium-expert-v2.pth b/halfcheetah_medium_expert_sparse/mbpo_based/realnvp/seed_456_0329_235735_halfcheetah_medium_expert_v2_gormpo_sparse_72.5/policy_halfcheetah-medium-expert-v2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/checkpoint/policy.pth b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/checkpoint/policy.pth new file mode 100644 index 0000000000000000000000000000000000000000..299de9b8c94a6011523d14426b1b40c2272c33a0 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/checkpoint/policy.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e994b9bae94794c02cfefafc9580ae00a414a0895eff669d26a437ce038e36ac +size 1459666 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/dynamics.pth b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/dynamics.pth new file mode 100644 index 0000000000000000000000000000000000000000..4c4fe7a50f266015682d7fb5f133994755c4a843 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/dynamics.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eed06b17da82e14ddd88b14cda8f4646029c68852f70b721c08d7ff626346ba9 +size 7435052 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/mu.npy b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/mu.npy new file mode 100644 index 0000000000000000000000000000000000000000..03884a3dcd9171c3b67cc1f93118cd39be81e790 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/mu.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2050ca9e9c5a2b446c05f0f06da81a2a3c08b258f520c8581b742e5edae5e1f +size 220 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/policy.pth b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/policy.pth new file mode 100644 index 0000000000000000000000000000000000000000..299de9b8c94a6011523d14426b1b40c2272c33a0 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/policy.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e994b9bae94794c02cfefafc9580ae00a414a0895eff669d26a437ce038e36ac +size 1459666 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/std.npy b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/std.npy new file mode 100644 index 0000000000000000000000000000000000000000..e01f32db015cdd35e5845d228d7d30dcafd00b65 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/model/std.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f34a2b6a9ea2b47097ce26e54fa7a4ad7a69196f7995bb6ea1c42783a6b230d +size 220 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/record/tb/events.out.tfevents.1778095935.aysin.1232333.0 b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/record/tb/events.out.tfevents.1778095935.aysin.1232333.0 new file mode 100644 index 0000000000000000000000000000000000000000..3e995fb547379fbe64f338f72d3c47792c0245ab --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193213/record/tb/events.out.tfevents.1778095935.aysin.1232333.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6a4821e56128649c9f1f98cfe0dfe1fc04fd2fb8bc9e31ee0a8a38737f582e2 +size 53332 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/checkpoint/policy.pth b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/checkpoint/policy.pth new file mode 100644 index 0000000000000000000000000000000000000000..a9ca783c8615c3f910379668eae80a2153328049 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/checkpoint/policy.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee22cda2c4832565c8e194c2eb6f05a33c4d34cd722ed414328866058a2f330c +size 1459666 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/dynamics.pth b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/dynamics.pth new file mode 100644 index 0000000000000000000000000000000000000000..e9c4825cfd3f269ef56a8ec429e94d2c9fbc9861 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/dynamics.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0257a72ba4f22219a6b9614b0f86c90ac99527bbff2086dee89d9e6ab570292 +size 7435052 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/mu.npy b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/mu.npy new file mode 100644 index 0000000000000000000000000000000000000000..84e3f305629a6450584c210eff1b0aa62e6f1954 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/mu.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30bd2b9e32a28f3d98dd1ab014db7c4a9b2577454213dc1874a070d93fd9db6a +size 220 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/policy.pth b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/policy.pth new file mode 100644 index 0000000000000000000000000000000000000000..a9ca783c8615c3f910379668eae80a2153328049 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/policy.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee22cda2c4832565c8e194c2eb6f05a33c4d34cd722ed414328866058a2f330c +size 1459666 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/std.npy b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/std.npy new file mode 100644 index 0000000000000000000000000000000000000000..1f912ff4dcde47510d7658a8c26f5c52237517be --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/model/std.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99eb0442dfec507acdf16ce545c8aa3fd2f9d9a9ba841f2477f7362c855c9dda +size 220 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/record/tb/events.out.tfevents.1778095935.aysin.1232334.0 b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/record/tb/events.out.tfevents.1778095935.aysin.1232334.0 new file mode 100644 index 0000000000000000000000000000000000000000..077d9789fed80973901db2a3273a14c772775f0a --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193213/record/tb/events.out.tfevents.1778095935.aysin.1232334.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eddcfa8ff05a2ddafc711e7a4a20220f1ba6dda0b1fdaa4ba3abf362c608e4e +size 53332 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-122101/record/consoleout_backup.txt b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-122101/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..808c77055c8047ad0ba3d633268e66aefc11a281 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-122101/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 250000, reward mean: 8.6452 +num rollout transitions: 250000, reward mean: 8.8035 +num rollout transitions: 250000, reward mean: 8.7932 +num rollout transitions: 250000, reward mean: 8.8001 +num rollout transitions: 250000, reward mean: 8.7752 +num rollout transitions: 250000, reward mean: 8.7991 +num rollout transitions: 250000, reward mean: 8.8056 +num rollout transitions: 250000, reward mean: 8.7789 +num rollout transitions: 250000, reward mean: 8.8117 +num rollout transitions: 250000, reward mean: 8.8454 +num rollout transitions: 250000, reward mean: 8.7956 +num rollout transitions: 250000, reward mean: 8.7686 +num rollout transitions: 250000, reward mean: 8.7937 +num rollout transitions: 250000, reward mean: 8.8288 +num rollout transitions: 250000, reward mean: 8.8757 +num rollout transitions: 250000, reward mean: 8.8743 +num rollout transitions: 250000, reward mean: 8.9091 +num rollout transitions: 250000, reward mean: 8.8124 +num rollout transitions: 250000, reward mean: 8.8176 +num rollout transitions: 250000, reward mean: 8.7787 +num rollout transitions: 250000, reward mean: 8.8085 +num rollout transitions: 250000, reward mean: 8.8971 +num rollout transitions: 250000, reward mean: 8.8750 +num rollout transitions: 250000, reward mean: 8.8244 +num rollout transitions: 250000, reward mean: 8.8449 +num rollout transitions: 250000, reward mean: 8.8531 +num rollout transitions: 250000, reward mean: 8.8941 +num rollout transitions: 250000, reward mean: 8.8543 +num rollout transitions: 250000, reward mean: 8.8717 +num rollout transitions: 250000, reward mean: 8.8695 +num rollout transitions: 250000, reward mean: 8.8999 +num rollout transitions: 250000, reward mean: 8.8512 +num rollout transitions: 250000, reward mean: 8.8541 +num rollout transitions: 250000, reward mean: 8.8515 +num rollout transitions: 250000, reward mean: 8.9173 +num rollout transitions: 250000, reward mean: 8.8698 +num rollout transitions: 250000, reward mean: 8.9249 +num rollout transitions: 250000, reward mean: 8.8719 +num rollout transitions: 250000, reward mean: 8.8636 +num rollout transitions: 250000, reward mean: 8.8129 +num rollout transitions: 250000, reward mean: 8.9282 +num rollout transitions: 250000, reward mean: 8.8666 +num rollout transitions: 250000, reward mean: 8.8679 +num rollout transitions: 250000, reward mean: 8.8893 +num rollout transitions: 250000, reward mean: 8.9163 +num rollout transitions: 250000, reward mean: 8.7727 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.8778 +num rollout transitions: 250000, reward mean: 8.8959 +num rollout transitions: 250000, reward mean: 8.9627 +---------------------------------------------------------------------------------- +| alpha | 0.509 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 24.6 | +| eval/normalized_episode_reward_std | 19.5 | +| loss/actor | -392 | +| loss/alpha | -0.351 | +| loss/critic | 263 | +| penalty/lcb_mean | 2.56 | +| penalty/lcb_std | 3.77 | +| rollout_info/dynamics_penalty_mean | 0.000269 | +| rollout_info/dynamics_penalty_std | 0.0157 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.85 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.8815 +num rollout transitions: 250000, reward mean: 8.8597 +num rollout transitions: 250000, reward mean: 8.9325 +num rollout transitions: 250000, reward mean: 8.9069 +num rollout transitions: 250000, reward mean: 8.9013 +num rollout transitions: 250000, reward mean: 8.8411 +num rollout transitions: 250000, reward mean: 8.8791 +num rollout transitions: 250000, reward mean: 8.8787 +num rollout transitions: 250000, reward mean: 8.8887 +num rollout transitions: 250000, reward mean: 8.8868 +num rollout transitions: 250000, reward mean: 8.9649 +num rollout transitions: 250000, reward mean: 8.8628 +num rollout transitions: 250000, reward mean: 8.8846 +num rollout transitions: 250000, reward mean: 8.8848 +num rollout transitions: 250000, reward mean: 8.9083 +num rollout transitions: 250000, reward mean: 8.8714 +num rollout transitions: 250000, reward mean: 8.9410 +num rollout transitions: 250000, reward mean: 8.9134 +num rollout transitions: 250000, reward mean: 8.8810 +num rollout transitions: 250000, reward mean: 8.9150 +num rollout transitions: 250000, reward mean: 8.8783 +num rollout transitions: 250000, reward mean: 8.9354 +num rollout transitions: 250000, reward mean: 8.8646 +num rollout transitions: 250000, reward mean: 8.8648 +num rollout transitions: 250000, reward mean: 8.8640 +num rollout transitions: 250000, reward mean: 8.8984 +num rollout transitions: 250000, reward mean: 8.9291 +num rollout transitions: 250000, reward mean: 8.9207 +num rollout transitions: 250000, reward mean: 8.8700 +num rollout transitions: 250000, reward mean: 8.9001 +num rollout transitions: 250000, reward mean: 8.8526 +num rollout transitions: 250000, reward mean: 8.8947 +num rollout transitions: 250000, reward mean: 8.9287 +num rollout transitions: 250000, reward mean: 8.9542 +num rollout transitions: 250000, reward mean: 8.9053 +num rollout transitions: 250000, reward mean: 8.9577 +num rollout transitions: 250000, reward mean: 8.8937 +num rollout transitions: 250000, reward mean: 8.8570 +num rollout transitions: 250000, reward mean: 8.9079 +num rollout transitions: 250000, reward mean: 8.8860 +num rollout transitions: 250000, reward mean: 8.9138 +num rollout transitions: 250000, reward mean: 8.9369 +num rollout transitions: 250000, reward mean: 8.9328 +num rollout transitions: 250000, reward mean: 8.8886 +num rollout transitions: 250000, reward mean: 8.9268 +num rollout transitions: 250000, reward mean: 8.8834 +num rollout transitions: 250000, reward mean: 8.8973 +num rollout transitions: 250000, reward mean: 8.8891 +num rollout transitions: 250000, reward mean: 8.8991 +num rollout transitions: 250000, reward mean: 8.9260 +---------------------------------------------------------------------------------- +| alpha | 0.653 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 38.2 | +| eval/normalized_episode_reward_std | 23.7 | +| loss/actor | -565 | +| loss/alpha | 0.00319 | +| loss/critic | 264 | +| penalty/lcb_mean | 2.97 | +| penalty/lcb_std | 3.82 | +| rollout_info/dynamics_penalty_mean | 0.000261 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.9 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.8763 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9463 +num rollout transitions: 250000, reward mean: 8.9144 +num rollout transitions: 250000, reward mean: 8.9171 +num rollout transitions: 250000, reward mean: 8.9481 +num rollout transitions: 250000, reward mean: 8.8766 +num rollout transitions: 250000, reward mean: 8.8788 +num rollout transitions: 250000, reward mean: 8.8978 +num rollout transitions: 250000, reward mean: 8.9342 +num rollout transitions: 250000, reward mean: 8.8895 +num rollout transitions: 250000, reward mean: 8.9191 +num rollout transitions: 250000, reward mean: 8.8978 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9261 +num rollout transitions: 250000, reward mean: 8.9060 +num rollout transitions: 250000, reward mean: 8.8706 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.8949 +num rollout transitions: 250000, reward mean: 8.8976 +num rollout transitions: 250000, reward mean: 8.8570 +num rollout transitions: 250000, reward mean: 8.9103 +num rollout transitions: 250000, reward mean: 8.9320 +num rollout transitions: 250000, reward mean: 8.9230 +num rollout transitions: 250000, reward mean: 8.9031 +num rollout transitions: 250000, reward mean: 8.9276 +num rollout transitions: 250000, reward mean: 8.8985 +num rollout transitions: 250000, reward mean: 8.8962 +num rollout transitions: 250000, reward mean: 8.8794 +num rollout transitions: 250000, reward mean: 8.9487 +num rollout transitions: 250000, reward mean: 8.9132 +num rollout transitions: 250000, reward mean: 8.9090 +num rollout transitions: 250000, reward mean: 8.9202 +num rollout transitions: 250000, reward mean: 8.9094 +num rollout transitions: 250000, reward mean: 8.9150 +num rollout transitions: 250000, reward mean: 8.9347 +num rollout transitions: 250000, reward mean: 8.8913 +num rollout transitions: 250000, reward mean: 8.8994 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.8891 +num rollout transitions: 250000, reward mean: 8.9731 +num rollout transitions: 250000, reward mean: 8.8657 +num rollout transitions: 250000, reward mean: 8.9271 +num rollout transitions: 250000, reward mean: 8.8920 +num rollout transitions: 250000, reward mean: 8.9085 +num rollout transitions: 250000, reward mean: 8.9572 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9027 +---------------------------------------------------------------------------------- +| alpha | 0.637 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 48.8 | +| eval/normalized_episode_reward_std | 27.3 | +| loss/actor | -700 | +| loss/alpha | -0.00094 | +| loss/critic | 218 | +| penalty/lcb_mean | 2.81 | +| penalty/lcb_std | 3.71 | +| rollout_info/dynamics_penalty_mean | 0.000267 | +| rollout_info/dynamics_penalty_std | 0.0157 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.91 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.8613 +num rollout transitions: 250000, reward mean: 8.9301 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.9368 +num rollout transitions: 250000, reward mean: 8.8942 +num rollout transitions: 250000, reward mean: 8.9333 +num rollout transitions: 250000, reward mean: 8.8875 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9680 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9071 +num rollout transitions: 250000, reward mean: 8.9086 +num rollout transitions: 250000, reward mean: 8.9040 +num rollout transitions: 250000, reward mean: 8.9080 +num rollout transitions: 250000, reward mean: 8.8974 +num rollout transitions: 250000, reward mean: 8.8878 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9345 +num rollout transitions: 250000, reward mean: 8.9115 +num rollout transitions: 250000, reward mean: 8.8933 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.9133 +num rollout transitions: 250000, reward mean: 8.9100 +num rollout transitions: 250000, reward mean: 8.8936 +num rollout transitions: 250000, reward mean: 8.9280 +num rollout transitions: 250000, reward mean: 8.9252 +num rollout transitions: 250000, reward mean: 8.9014 +num rollout transitions: 250000, reward mean: 8.8688 +num rollout transitions: 250000, reward mean: 8.9059 +num rollout transitions: 250000, reward mean: 8.9389 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9216 +num rollout transitions: 250000, reward mean: 8.8964 +num rollout transitions: 250000, reward mean: 8.8878 +num rollout transitions: 250000, reward mean: 8.8760 +num rollout transitions: 250000, reward mean: 8.9200 +num rollout transitions: 250000, reward mean: 8.9097 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9381 +num rollout transitions: 250000, reward mean: 8.9509 +num rollout transitions: 250000, reward mean: 8.8980 +num rollout transitions: 250000, reward mean: 8.8992 +num rollout transitions: 250000, reward mean: 8.9237 +num rollout transitions: 250000, reward mean: 8.8942 +num rollout transitions: 250000, reward mean: 8.9096 +num rollout transitions: 250000, reward mean: 8.9271 +num rollout transitions: 250000, reward mean: 8.9501 +num rollout transitions: 250000, reward mean: 8.9414 +num rollout transitions: 250000, reward mean: 8.9257 +---------------------------------------------------------------------------------- +| alpha | 0.596 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 60 | +| eval/normalized_episode_reward_std | 30.4 | +| loss/actor | -781 | +| loss/alpha | -0.00334 | +| loss/critic | 184 | +| penalty/lcb_mean | 2.58 | +| penalty/lcb_std | 3.58 | +| rollout_info/dynamics_penalty_mean | 0.000275 | +| rollout_info/dynamics_penalty_std | 0.0159 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.92 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.8747 +num rollout transitions: 250000, reward mean: 8.9075 +num rollout transitions: 250000, reward mean: 8.9160 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.9272 +num rollout transitions: 250000, reward mean: 8.9105 +num rollout transitions: 250000, reward mean: 8.9041 +num rollout transitions: 250000, reward mean: 8.9118 +num rollout transitions: 250000, reward mean: 8.9140 +num rollout transitions: 250000, reward mean: 8.9112 +num rollout transitions: 250000, reward mean: 8.9157 +num rollout transitions: 250000, reward mean: 8.9305 +num rollout transitions: 250000, reward mean: 8.9235 +num rollout transitions: 250000, reward mean: 8.9024 +num rollout transitions: 250000, reward mean: 8.9287 +num rollout transitions: 250000, reward mean: 8.8664 +num rollout transitions: 250000, reward mean: 8.9303 +num rollout transitions: 250000, reward mean: 8.9114 +num rollout transitions: 250000, reward mean: 8.9732 +num rollout transitions: 250000, reward mean: 8.9764 +num rollout transitions: 250000, reward mean: 8.9066 +num rollout transitions: 250000, reward mean: 8.9080 +num rollout transitions: 250000, reward mean: 8.8583 +num rollout transitions: 250000, reward mean: 8.8981 +num rollout transitions: 250000, reward mean: 8.9052 +num rollout transitions: 250000, reward mean: 8.9326 +num rollout transitions: 250000, reward mean: 8.9173 +num rollout transitions: 250000, reward mean: 8.9522 +num rollout transitions: 250000, reward mean: 8.9023 +num rollout transitions: 250000, reward mean: 8.8810 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.8938 +num rollout transitions: 250000, reward mean: 8.9216 +num rollout transitions: 250000, reward mean: 8.9129 +num rollout transitions: 250000, reward mean: 8.9062 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9312 +num rollout transitions: 250000, reward mean: 8.9110 +num rollout transitions: 250000, reward mean: 8.9466 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9194 +num rollout transitions: 250000, reward mean: 8.8915 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9349 +num rollout transitions: 250000, reward mean: 8.9269 +num rollout transitions: 250000, reward mean: 8.9384 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9127 +num rollout transitions: 250000, reward mean: 8.9347 +---------------------------------------------------------------------------------- +| alpha | 0.54 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 95.4 | +| eval/normalized_episode_reward_std | 7.54 | +| loss/actor | -845 | +| loss/alpha | -0.00383 | +| loss/critic | 150 | +| penalty/lcb_mean | 2.31 | +| penalty/lcb_std | 3.32 | +| rollout_info/dynamics_penalty_mean | 0.00026 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.92 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9043 +num rollout transitions: 250000, reward mean: 8.9345 +num rollout transitions: 250000, reward mean: 8.8981 +num rollout transitions: 250000, reward mean: 8.9218 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.8465 +num rollout transitions: 250000, reward mean: 8.9184 +num rollout transitions: 250000, reward mean: 8.9199 +num rollout transitions: 250000, reward mean: 8.9022 +num rollout transitions: 250000, reward mean: 8.9438 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9158 +num rollout transitions: 250000, reward mean: 8.8904 +num rollout transitions: 250000, reward mean: 8.9251 +num rollout transitions: 250000, reward mean: 8.8996 +num rollout transitions: 250000, reward mean: 8.9360 +num rollout transitions: 250000, reward mean: 8.9050 +num rollout transitions: 250000, reward mean: 8.8899 +num rollout transitions: 250000, reward mean: 8.9171 +num rollout transitions: 250000, reward mean: 8.9413 +num rollout transitions: 250000, reward mean: 8.9102 +num rollout transitions: 250000, reward mean: 8.8980 +num rollout transitions: 250000, reward mean: 8.9290 +num rollout transitions: 250000, reward mean: 8.9296 +num rollout transitions: 250000, reward mean: 8.9192 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9149 +num rollout transitions: 250000, reward mean: 8.9149 +num rollout transitions: 250000, reward mean: 8.9401 +num rollout transitions: 250000, reward mean: 8.9328 +num rollout transitions: 250000, reward mean: 8.9343 +num rollout transitions: 250000, reward mean: 8.9461 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9188 +num rollout transitions: 250000, reward mean: 8.9293 +num rollout transitions: 250000, reward mean: 8.9155 +num rollout transitions: 250000, reward mean: 8.9107 +num rollout transitions: 250000, reward mean: 8.9023 +num rollout transitions: 250000, reward mean: 8.9551 +num rollout transitions: 250000, reward mean: 8.9354 +num rollout transitions: 250000, reward mean: 8.8926 +num rollout transitions: 250000, reward mean: 8.9187 +num rollout transitions: 250000, reward mean: 8.8920 +num rollout transitions: 250000, reward mean: 8.9209 +num rollout transitions: 250000, reward mean: 8.9162 +num rollout transitions: 250000, reward mean: 8.9014 +num rollout transitions: 250000, reward mean: 8.8733 +---------------------------------------------------------------------------------- +| alpha | 0.499 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 93.4 | +| eval/normalized_episode_reward_std | 14.8 | +| loss/actor | -875 | +| loss/alpha | -0.0016 | +| loss/critic | 132 | +| penalty/lcb_mean | 2.15 | +| penalty/lcb_std | 3.19 | +| rollout_info/dynamics_penalty_mean | 0.000262 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.92 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.8970 +num rollout transitions: 250000, reward mean: 8.9668 +num rollout transitions: 250000, reward mean: 8.9273 +num rollout transitions: 250000, reward mean: 8.9258 +num rollout transitions: 250000, reward mean: 8.8939 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9153 +num rollout transitions: 250000, reward mean: 8.9307 +num rollout transitions: 250000, reward mean: 8.9194 +num rollout transitions: 250000, reward mean: 8.9101 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.8988 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9452 +num rollout transitions: 250000, reward mean: 8.9502 +num rollout transitions: 250000, reward mean: 8.9228 +num rollout transitions: 250000, reward mean: 8.9273 +num rollout transitions: 250000, reward mean: 8.9076 +num rollout transitions: 250000, reward mean: 8.9068 +num rollout transitions: 250000, reward mean: 8.9282 +num rollout transitions: 250000, reward mean: 8.9120 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9182 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9100 +num rollout transitions: 250000, reward mean: 8.8943 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9350 +num rollout transitions: 250000, reward mean: 8.9055 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.8810 +num rollout transitions: 250000, reward mean: 8.9195 +num rollout transitions: 250000, reward mean: 8.9016 +num rollout transitions: 250000, reward mean: 8.9556 +num rollout transitions: 250000, reward mean: 8.9349 +num rollout transitions: 250000, reward mean: 8.9164 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9228 +num rollout transitions: 250000, reward mean: 8.9237 +num rollout transitions: 250000, reward mean: 8.9146 +num rollout transitions: 250000, reward mean: 8.9158 +num rollout transitions: 250000, reward mean: 8.9004 +num rollout transitions: 250000, reward mean: 8.9254 +num rollout transitions: 250000, reward mean: 8.9156 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.8983 +num rollout transitions: 250000, reward mean: 8.8857 +num rollout transitions: 250000, reward mean: 8.8901 +---------------------------------------------------------------------------------- +| alpha | 0.473 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 94.8 | +| eval/normalized_episode_reward_std | 11.5 | +| loss/actor | -900 | +| loss/alpha | -0.00208 | +| loss/critic | 114 | +| penalty/lcb_mean | 2.02 | +| penalty/lcb_std | 3 | +| rollout_info/dynamics_penalty_mean | 0.000282 | +| rollout_info/dynamics_penalty_std | 0.0161 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.92 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9204 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9188 +num rollout transitions: 250000, reward mean: 8.9066 +num rollout transitions: 250000, reward mean: 8.9146 +num rollout transitions: 250000, reward mean: 8.9273 +num rollout transitions: 250000, reward mean: 8.9414 +num rollout transitions: 250000, reward mean: 8.9384 +num rollout transitions: 250000, reward mean: 8.9303 +num rollout transitions: 250000, reward mean: 8.9073 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9303 +num rollout transitions: 250000, reward mean: 8.9390 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9253 +num rollout transitions: 250000, reward mean: 8.9282 +num rollout transitions: 250000, reward mean: 8.9112 +num rollout transitions: 250000, reward mean: 8.9464 +num rollout transitions: 250000, reward mean: 8.9477 +num rollout transitions: 250000, reward mean: 8.9030 +num rollout transitions: 250000, reward mean: 8.8888 +num rollout transitions: 250000, reward mean: 8.8860 +num rollout transitions: 250000, reward mean: 8.9345 +num rollout transitions: 250000, reward mean: 8.9300 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9080 +num rollout transitions: 250000, reward mean: 8.9107 +num rollout transitions: 250000, reward mean: 8.9273 +num rollout transitions: 250000, reward mean: 8.8828 +num rollout transitions: 250000, reward mean: 8.9290 +num rollout transitions: 250000, reward mean: 8.9431 +num rollout transitions: 250000, reward mean: 8.9232 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9157 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9349 +num rollout transitions: 250000, reward mean: 8.9088 +num rollout transitions: 250000, reward mean: 8.9337 +num rollout transitions: 250000, reward mean: 8.9713 +num rollout transitions: 250000, reward mean: 8.8670 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.8970 +num rollout transitions: 250000, reward mean: 8.9240 +num rollout transitions: 250000, reward mean: 8.9270 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9278 +num rollout transitions: 250000, reward mean: 8.9468 +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.9455 +num rollout transitions: 250000, reward mean: 8.9705 +---------------------------------------------------------------------------------- +| alpha | 0.453 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 96.1 | +| eval/normalized_episode_reward_std | 11 | +| loss/actor | -917 | +| loss/alpha | -0.00201 | +| loss/critic | 109 | +| penalty/lcb_mean | 1.96 | +| penalty/lcb_std | 2.94 | +| rollout_info/dynamics_penalty_mean | 0.000268 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9573 +num rollout transitions: 250000, reward mean: 8.9205 +num rollout transitions: 250000, reward mean: 8.9416 +num rollout transitions: 250000, reward mean: 8.9247 +num rollout transitions: 250000, reward mean: 8.9056 +num rollout transitions: 250000, reward mean: 8.9065 +num rollout transitions: 250000, reward mean: 8.8792 +num rollout transitions: 250000, reward mean: 8.9199 +num rollout transitions: 250000, reward mean: 8.9403 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9258 +num rollout transitions: 250000, reward mean: 8.9256 +num rollout transitions: 250000, reward mean: 8.9275 +num rollout transitions: 250000, reward mean: 8.9706 +num rollout transitions: 250000, reward mean: 8.8982 +num rollout transitions: 250000, reward mean: 8.9009 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9163 +num rollout transitions: 250000, reward mean: 8.9251 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9096 +num rollout transitions: 250000, reward mean: 8.9679 +num rollout transitions: 250000, reward mean: 8.9254 +num rollout transitions: 250000, reward mean: 8.9010 +num rollout transitions: 250000, reward mean: 8.9179 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.9851 +num rollout transitions: 250000, reward mean: 8.9295 +num rollout transitions: 250000, reward mean: 8.9345 +num rollout transitions: 250000, reward mean: 8.9126 +num rollout transitions: 250000, reward mean: 8.9129 +num rollout transitions: 250000, reward mean: 8.9136 +num rollout transitions: 250000, reward mean: 8.9338 +num rollout transitions: 250000, reward mean: 8.9571 +num rollout transitions: 250000, reward mean: 8.9269 +num rollout transitions: 250000, reward mean: 8.9396 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9076 +num rollout transitions: 250000, reward mean: 8.9178 +num rollout transitions: 250000, reward mean: 8.9073 +num rollout transitions: 250000, reward mean: 8.9051 +num rollout transitions: 250000, reward mean: 8.9376 +num rollout transitions: 250000, reward mean: 8.9279 +num rollout transitions: 250000, reward mean: 8.8963 +num rollout transitions: 250000, reward mean: 8.9030 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.9039 +num rollout transitions: 250000, reward mean: 8.9474 +num rollout transitions: 250000, reward mean: 8.9295 +num rollout transitions: 250000, reward mean: 8.9300 +---------------------------------------------------------------------------------- +| alpha | 0.435 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 88.5 | +| eval/normalized_episode_reward_std | 11.4 | +| loss/actor | -921 | +| loss/alpha | -0.00152 | +| loss/critic | 109 | +| penalty/lcb_mean | 1.94 | +| penalty/lcb_std | 3 | +| rollout_info/dynamics_penalty_mean | 0.000261 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9275 +num rollout transitions: 250000, reward mean: 8.9220 +num rollout transitions: 250000, reward mean: 8.8870 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9384 +num rollout transitions: 250000, reward mean: 8.9551 +num rollout transitions: 250000, reward mean: 8.8897 +num rollout transitions: 250000, reward mean: 8.9572 +num rollout transitions: 250000, reward mean: 8.9265 +num rollout transitions: 250000, reward mean: 8.9211 +num rollout transitions: 250000, reward mean: 8.9522 +num rollout transitions: 250000, reward mean: 8.9180 +num rollout transitions: 250000, reward mean: 8.8991 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9260 +num rollout transitions: 250000, reward mean: 8.9041 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9755 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9061 +num rollout transitions: 250000, reward mean: 8.9367 +num rollout transitions: 250000, reward mean: 8.9075 +num rollout transitions: 250000, reward mean: 8.9525 +num rollout transitions: 250000, reward mean: 8.9225 +num rollout transitions: 250000, reward mean: 8.9056 +num rollout transitions: 250000, reward mean: 8.9461 +num rollout transitions: 250000, reward mean: 8.9164 +num rollout transitions: 250000, reward mean: 8.9701 +num rollout transitions: 250000, reward mean: 8.9665 +num rollout transitions: 250000, reward mean: 8.9178 +num rollout transitions: 250000, reward mean: 8.9172 +num rollout transitions: 250000, reward mean: 8.8941 +num rollout transitions: 250000, reward mean: 8.9013 +num rollout transitions: 250000, reward mean: 8.9347 +num rollout transitions: 250000, reward mean: 8.9103 +num rollout transitions: 250000, reward mean: 8.9466 +num rollout transitions: 250000, reward mean: 8.9367 +num rollout transitions: 250000, reward mean: 8.9439 +num rollout transitions: 250000, reward mean: 8.9258 +num rollout transitions: 250000, reward mean: 8.9620 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9506 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9350 +num rollout transitions: 250000, reward mean: 8.9685 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9108 +num rollout transitions: 250000, reward mean: 8.9265 +num rollout transitions: 250000, reward mean: 8.9394 +num rollout transitions: 250000, reward mean: 8.9153 +----------------------------------------------------------------------------------- +| alpha | 0.423 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 93.7 | +| eval/normalized_episode_reward_std | 15.7 | +| loss/actor | -930 | +| loss/alpha | -0.000604 | +| loss/critic | 105 | +| penalty/lcb_mean | 1.9 | +| penalty/lcb_std | 3 | +| rollout_info/dynamics_penalty_mean | 0.000268 | +| rollout_info/dynamics_penalty_std | 0.0157 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 500000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9355 +num rollout transitions: 250000, reward mean: 8.9190 +num rollout transitions: 250000, reward mean: 8.9867 +num rollout transitions: 250000, reward mean: 8.9620 +num rollout transitions: 250000, reward mean: 8.9517 +num rollout transitions: 250000, reward mean: 8.9581 +num rollout transitions: 250000, reward mean: 8.9199 +num rollout transitions: 250000, reward mean: 8.9267 +num rollout transitions: 250000, reward mean: 8.9561 +num rollout transitions: 250000, reward mean: 8.9538 +num rollout transitions: 250000, reward mean: 8.9205 +num rollout transitions: 250000, reward mean: 8.9205 +num rollout transitions: 250000, reward mean: 8.9621 +num rollout transitions: 250000, reward mean: 8.9107 +num rollout transitions: 250000, reward mean: 8.9586 +num rollout transitions: 250000, reward mean: 8.9480 +num rollout transitions: 250000, reward mean: 8.9243 +num rollout transitions: 250000, reward mean: 8.9325 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9229 +num rollout transitions: 250000, reward mean: 8.9717 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9360 +num rollout transitions: 250000, reward mean: 8.9081 +num rollout transitions: 250000, reward mean: 8.9046 +num rollout transitions: 250000, reward mean: 8.9483 +num rollout transitions: 250000, reward mean: 8.9341 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9680 +num rollout transitions: 250000, reward mean: 8.9035 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9130 +num rollout transitions: 250000, reward mean: 8.9202 +num rollout transitions: 250000, reward mean: 8.9503 +num rollout transitions: 250000, reward mean: 8.9565 +num rollout transitions: 250000, reward mean: 8.9184 +num rollout transitions: 250000, reward mean: 8.9195 +num rollout transitions: 250000, reward mean: 8.9420 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9226 +num rollout transitions: 250000, reward mean: 8.9429 +num rollout transitions: 250000, reward mean: 8.9209 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9687 +num rollout transitions: 250000, reward mean: 8.9436 +num rollout transitions: 250000, reward mean: 8.9085 +num rollout transitions: 250000, reward mean: 8.9384 +---------------------------------------------------------------------------------- +| alpha | 0.416 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 95.8 | +| eval/normalized_episode_reward_std | 11 | +| loss/actor | -940 | +| loss/alpha | 0.00101 | +| loss/critic | 97.7 | +| penalty/lcb_mean | 1.85 | +| penalty/lcb_std | 2.91 | +| rollout_info/dynamics_penalty_mean | 0.000281 | +| rollout_info/dynamics_penalty_std | 0.0161 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9278 +num rollout transitions: 250000, reward mean: 8.9419 +num rollout transitions: 250000, reward mean: 8.9727 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9172 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.8962 +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9264 +num rollout transitions: 250000, reward mean: 8.9442 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9128 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9288 +num rollout transitions: 250000, reward mean: 8.9410 +num rollout transitions: 250000, reward mean: 8.8929 +num rollout transitions: 250000, reward mean: 8.9298 +num rollout transitions: 250000, reward mean: 8.9619 +num rollout transitions: 250000, reward mean: 8.9239 +num rollout transitions: 250000, reward mean: 8.9087 +num rollout transitions: 250000, reward mean: 8.9472 +num rollout transitions: 250000, reward mean: 8.9396 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9279 +num rollout transitions: 250000, reward mean: 8.9026 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9484 +num rollout transitions: 250000, reward mean: 8.9571 +num rollout transitions: 250000, reward mean: 8.9552 +num rollout transitions: 250000, reward mean: 8.9236 +num rollout transitions: 250000, reward mean: 8.8907 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9086 +num rollout transitions: 250000, reward mean: 8.9331 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9619 +num rollout transitions: 250000, reward mean: 8.9998 +num rollout transitions: 250000, reward mean: 8.8977 +num rollout transitions: 250000, reward mean: 8.9009 +num rollout transitions: 250000, reward mean: 8.9677 +num rollout transitions: 250000, reward mean: 8.9229 +num rollout transitions: 250000, reward mean: 8.9439 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9021 +num rollout transitions: 250000, reward mean: 8.9672 +num rollout transitions: 250000, reward mean: 8.9042 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9544 +---------------------------------------------------------------------------------- +| alpha | 0.404 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 97.5 | +| eval/normalized_episode_reward_std | 11.1 | +| loss/actor | -941 | +| loss/alpha | -0.00288 | +| loss/critic | 96.8 | +| penalty/lcb_mean | 1.83 | +| penalty/lcb_std | 2.88 | +| rollout_info/dynamics_penalty_mean | 0.000256 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9442 +num rollout transitions: 250000, reward mean: 8.9513 +num rollout transitions: 250000, reward mean: 8.9457 +num rollout transitions: 250000, reward mean: 8.9442 +num rollout transitions: 250000, reward mean: 8.9225 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9565 +num rollout transitions: 250000, reward mean: 8.9230 +num rollout transitions: 250000, reward mean: 8.9234 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9119 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9379 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9123 +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.9157 +num rollout transitions: 250000, reward mean: 8.9562 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.9656 +num rollout transitions: 250000, reward mean: 8.9329 +num rollout transitions: 250000, reward mean: 8.9219 +num rollout transitions: 250000, reward mean: 8.9302 +num rollout transitions: 250000, reward mean: 8.9682 +num rollout transitions: 250000, reward mean: 8.9243 +num rollout transitions: 250000, reward mean: 8.9242 +num rollout transitions: 250000, reward mean: 8.9171 +num rollout transitions: 250000, reward mean: 8.9569 +num rollout transitions: 250000, reward mean: 8.9011 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9642 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.9614 +num rollout transitions: 250000, reward mean: 8.9281 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9263 +num rollout transitions: 250000, reward mean: 8.9261 +num rollout transitions: 250000, reward mean: 8.9669 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9204 +num rollout transitions: 250000, reward mean: 8.9235 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.8980 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.9557 +---------------------------------------------------------------------------------- +| alpha | 0.4 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 95 | +| eval/normalized_episode_reward_std | 17.9 | +| loss/actor | -948 | +| loss/alpha | 0.000577 | +| loss/critic | 94.5 | +| penalty/lcb_mean | 1.81 | +| penalty/lcb_std | 2.87 | +| rollout_info/dynamics_penalty_mean | 0.000275 | +| rollout_info/dynamics_penalty_std | 0.016 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9227 +num rollout transitions: 250000, reward mean: 8.9691 +num rollout transitions: 250000, reward mean: 8.9498 +num rollout transitions: 250000, reward mean: 8.9030 +num rollout transitions: 250000, reward mean: 8.9421 +num rollout transitions: 250000, reward mean: 8.9568 +num rollout transitions: 250000, reward mean: 8.9624 +num rollout transitions: 250000, reward mean: 8.9503 +num rollout transitions: 250000, reward mean: 8.9086 +num rollout transitions: 250000, reward mean: 8.9414 +num rollout transitions: 250000, reward mean: 8.9364 +num rollout transitions: 250000, reward mean: 8.9135 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9731 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9474 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.9143 +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.8872 +num rollout transitions: 250000, reward mean: 8.9177 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9177 +num rollout transitions: 250000, reward mean: 8.9474 +num rollout transitions: 250000, reward mean: 8.9204 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9681 +num rollout transitions: 250000, reward mean: 8.9449 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.9575 +num rollout transitions: 250000, reward mean: 8.8958 +num rollout transitions: 250000, reward mean: 8.9180 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9641 +num rollout transitions: 250000, reward mean: 8.9312 +num rollout transitions: 250000, reward mean: 8.9625 +num rollout transitions: 250000, reward mean: 8.9135 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9251 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9683 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9265 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.9635 +num rollout transitions: 250000, reward mean: 8.9102 +----------------------------------------------------------------------------------- +| alpha | 0.393 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 92.3 | +| eval/normalized_episode_reward_std | 3.5 | +| loss/actor | -951 | +| loss/alpha | -0.000557 | +| loss/critic | 92.8 | +| penalty/lcb_mean | 1.8 | +| penalty/lcb_std | 2.84 | +| rollout_info/dynamics_penalty_mean | 0.000271 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 700000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.8962 +num rollout transitions: 250000, reward mean: 8.9345 +num rollout transitions: 250000, reward mean: 8.9079 +num rollout transitions: 250000, reward mean: 8.9494 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9177 +num rollout transitions: 250000, reward mean: 8.9532 +num rollout transitions: 250000, reward mean: 8.9590 +num rollout transitions: 250000, reward mean: 8.9202 +num rollout transitions: 250000, reward mean: 8.8972 +num rollout transitions: 250000, reward mean: 8.9298 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9676 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9733 +num rollout transitions: 250000, reward mean: 8.9276 +num rollout transitions: 250000, reward mean: 8.9309 +num rollout transitions: 250000, reward mean: 8.9234 +num rollout transitions: 250000, reward mean: 8.8895 +num rollout transitions: 250000, reward mean: 8.9669 +num rollout transitions: 250000, reward mean: 8.9171 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9209 +num rollout transitions: 250000, reward mean: 8.9132 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9582 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9588 +num rollout transitions: 250000, reward mean: 8.9484 +num rollout transitions: 250000, reward mean: 8.9417 +num rollout transitions: 250000, reward mean: 8.9244 +num rollout transitions: 250000, reward mean: 8.9269 +num rollout transitions: 250000, reward mean: 8.8970 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9176 +num rollout transitions: 250000, reward mean: 8.9269 +num rollout transitions: 250000, reward mean: 8.8994 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.9544 +num rollout transitions: 250000, reward mean: 8.9778 +num rollout transitions: 250000, reward mean: 8.9063 +num rollout transitions: 250000, reward mean: 8.9059 +num rollout transitions: 250000, reward mean: 8.9618 +num rollout transitions: 250000, reward mean: 8.9400 +---------------------------------------------------------------------------------- +| alpha | 0.39 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 93.3 | +| eval/normalized_episode_reward_std | 12.2 | +| loss/actor | -950 | +| loss/alpha | 0.000786 | +| loss/critic | 96.3 | +| penalty/lcb_mean | 1.82 | +| penalty/lcb_std | 2.94 | +| rollout_info/dynamics_penalty_mean | 0.000259 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9368 +num rollout transitions: 250000, reward mean: 8.9247 +num rollout transitions: 250000, reward mean: 8.9104 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9387 +num rollout transitions: 250000, reward mean: 8.9279 +num rollout transitions: 250000, reward mean: 8.9254 +num rollout transitions: 250000, reward mean: 8.9176 +num rollout transitions: 250000, reward mean: 8.9528 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9456 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9374 +num rollout transitions: 250000, reward mean: 8.9828 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9164 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9453 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9294 +num rollout transitions: 250000, reward mean: 8.9111 +num rollout transitions: 250000, reward mean: 8.9544 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9606 +num rollout transitions: 250000, reward mean: 8.9386 +num rollout transitions: 250000, reward mean: 8.8762 +num rollout transitions: 250000, reward mean: 8.9001 +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.9301 +num rollout transitions: 250000, reward mean: 8.9493 +num rollout transitions: 250000, reward mean: 8.9427 +num rollout transitions: 250000, reward mean: 8.9453 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9354 +num rollout transitions: 250000, reward mean: 8.9827 +num rollout transitions: 250000, reward mean: 8.9379 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9483 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9401 +num rollout transitions: 250000, reward mean: 8.9343 +num rollout transitions: 250000, reward mean: 8.9460 +---------------------------------------------------------------------------------- +| alpha | 0.387 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.6 | +| eval/normalized_episode_reward_std | 3.2 | +| loss/actor | -953 | +| loss/alpha | -0.00355 | +| loss/critic | 91.4 | +| penalty/lcb_mean | 1.77 | +| penalty/lcb_std | 2.87 | +| rollout_info/dynamics_penalty_mean | 0.000271 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9512 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9395 +num rollout transitions: 250000, reward mean: 8.9255 +num rollout transitions: 250000, reward mean: 8.9360 +num rollout transitions: 250000, reward mean: 8.9226 +num rollout transitions: 250000, reward mean: 8.9336 +num rollout transitions: 250000, reward mean: 8.9576 +num rollout transitions: 250000, reward mean: 8.9304 +num rollout transitions: 250000, reward mean: 8.9410 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9087 +num rollout transitions: 250000, reward mean: 8.9609 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9314 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9389 +num rollout transitions: 250000, reward mean: 8.9219 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9185 +num rollout transitions: 250000, reward mean: 8.9328 +num rollout transitions: 250000, reward mean: 8.9260 +num rollout transitions: 250000, reward mean: 8.9646 +num rollout transitions: 250000, reward mean: 8.9337 +num rollout transitions: 250000, reward mean: 8.9393 +num rollout transitions: 250000, reward mean: 8.9260 +num rollout transitions: 250000, reward mean: 8.9255 +num rollout transitions: 250000, reward mean: 8.9153 +num rollout transitions: 250000, reward mean: 8.9142 +num rollout transitions: 250000, reward mean: 8.9295 +num rollout transitions: 250000, reward mean: 8.9042 +num rollout transitions: 250000, reward mean: 8.9238 +num rollout transitions: 250000, reward mean: 8.9173 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9694 +num rollout transitions: 250000, reward mean: 8.9226 +num rollout transitions: 250000, reward mean: 8.9210 +num rollout transitions: 250000, reward mean: 8.9513 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9704 +num rollout transitions: 250000, reward mean: 8.9276 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9153 +num rollout transitions: 250000, reward mean: 8.9394 +num rollout transitions: 250000, reward mean: 8.9667 +num rollout transitions: 250000, reward mean: 8.9468 +---------------------------------------------------------------------------------- +| alpha | 0.385 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 87.7 | +| eval/normalized_episode_reward_std | 21.1 | +| loss/actor | -957 | +| loss/alpha | 0.00097 | +| loss/critic | 89.1 | +| penalty/lcb_mean | 1.76 | +| penalty/lcb_std | 2.81 | +| rollout_info/dynamics_penalty_mean | 0.000287 | +| rollout_info/dynamics_penalty_std | 0.0163 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9756 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.8972 +num rollout transitions: 250000, reward mean: 8.9222 +num rollout transitions: 250000, reward mean: 8.9812 +num rollout transitions: 250000, reward mean: 8.9648 +num rollout transitions: 250000, reward mean: 8.9578 +num rollout transitions: 250000, reward mean: 8.9316 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9256 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9436 +num rollout transitions: 250000, reward mean: 8.9268 +num rollout transitions: 250000, reward mean: 8.9646 +num rollout transitions: 250000, reward mean: 8.9158 +num rollout transitions: 250000, reward mean: 8.9259 +num rollout transitions: 250000, reward mean: 8.9536 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9505 +num rollout transitions: 250000, reward mean: 8.9291 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9272 +num rollout transitions: 250000, reward mean: 8.9369 +num rollout transitions: 250000, reward mean: 8.9093 +num rollout transitions: 250000, reward mean: 8.9305 +num rollout transitions: 250000, reward mean: 8.9161 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9508 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9622 +num rollout transitions: 250000, reward mean: 8.8819 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9690 +num rollout transitions: 250000, reward mean: 8.9025 +num rollout transitions: 250000, reward mean: 8.9133 +num rollout transitions: 250000, reward mean: 8.9592 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.8857 +num rollout transitions: 250000, reward mean: 8.9300 +num rollout transitions: 250000, reward mean: 8.9310 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9548 +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.9884 +num rollout transitions: 250000, reward mean: 8.9455 +num rollout transitions: 250000, reward mean: 8.9244 +num rollout transitions: 250000, reward mean: 8.9574 +----------------------------------------------------------------------------------- +| alpha | 0.381 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 100 | +| eval/normalized_episode_reward_std | 3.49 | +| loss/actor | -958 | +| loss/alpha | -0.000799 | +| loss/critic | 87.6 | +| penalty/lcb_mean | 1.75 | +| penalty/lcb_std | 2.78 | +| rollout_info/dynamics_penalty_mean | 0.000257 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 900000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9595 +num rollout transitions: 250000, reward mean: 8.9622 +num rollout transitions: 250000, reward mean: 8.9031 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9224 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9240 +num rollout transitions: 250000, reward mean: 8.9389 +num rollout transitions: 250000, reward mean: 8.9293 +num rollout transitions: 250000, reward mean: 8.9659 +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9298 +num rollout transitions: 250000, reward mean: 8.9486 +num rollout transitions: 250000, reward mean: 8.9558 +num rollout transitions: 250000, reward mean: 8.9301 +num rollout transitions: 250000, reward mean: 8.8928 +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9707 +num rollout transitions: 250000, reward mean: 8.9198 +num rollout transitions: 250000, reward mean: 8.9727 +num rollout transitions: 250000, reward mean: 8.9392 +num rollout transitions: 250000, reward mean: 8.9573 +num rollout transitions: 250000, reward mean: 8.9213 +num rollout transitions: 250000, reward mean: 8.9396 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9343 +num rollout transitions: 250000, reward mean: 8.9379 +num rollout transitions: 250000, reward mean: 8.9300 +num rollout transitions: 250000, reward mean: 8.9201 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9300 +num rollout transitions: 250000, reward mean: 8.9703 +num rollout transitions: 250000, reward mean: 8.9034 +num rollout transitions: 250000, reward mean: 8.9827 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9200 +num rollout transitions: 250000, reward mean: 8.9426 +num rollout transitions: 250000, reward mean: 8.9316 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9380 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.8884 +num rollout transitions: 250000, reward mean: 8.9644 +num rollout transitions: 250000, reward mean: 8.9470 +----------------------------------------------------------------------------------- +| alpha | 0.382 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 95.4 | +| eval/normalized_episode_reward_std | 16.6 | +| loss/actor | -960 | +| loss/alpha | -0.000761 | +| loss/critic | 87.6 | +| penalty/lcb_mean | 1.75 | +| penalty/lcb_std | 2.82 | +| rollout_info/dynamics_penalty_mean | 0.000293 | +| rollout_info/dynamics_penalty_std | 0.0164 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 950000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9262 +num rollout transitions: 250000, reward mean: 8.9358 +num rollout transitions: 250000, reward mean: 8.9199 +num rollout transitions: 250000, reward mean: 8.9766 +num rollout transitions: 250000, reward mean: 8.9656 +num rollout transitions: 250000, reward mean: 8.9368 +num rollout transitions: 250000, reward mean: 8.9156 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9668 +num rollout transitions: 250000, reward mean: 8.9184 +num rollout transitions: 250000, reward mean: 8.9578 +num rollout transitions: 250000, reward mean: 8.9099 +num rollout transitions: 250000, reward mean: 8.9208 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9294 +num rollout transitions: 250000, reward mean: 8.9885 +num rollout transitions: 250000, reward mean: 8.9320 +num rollout transitions: 250000, reward mean: 8.9767 +num rollout transitions: 250000, reward mean: 8.9211 +num rollout transitions: 250000, reward mean: 8.9625 +num rollout transitions: 250000, reward mean: 8.9571 +num rollout transitions: 250000, reward mean: 8.9459 +num rollout transitions: 250000, reward mean: 8.9288 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9600 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9476 +num rollout transitions: 250000, reward mean: 8.9069 +num rollout transitions: 250000, reward mean: 8.9403 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9226 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9577 +num rollout transitions: 250000, reward mean: 8.9297 +num rollout transitions: 250000, reward mean: 8.9260 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9107 +num rollout transitions: 250000, reward mean: 8.9379 +num rollout transitions: 250000, reward mean: 8.9945 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9848 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9167 +num rollout transitions: 250000, reward mean: 8.9462 +----------------------------------------------------------------------------------- +| alpha | 0.374 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.1 | +| eval/normalized_episode_reward_std | 3.26 | +| loss/actor | -962 | +| loss/alpha | -0.000944 | +| loss/critic | 84.4 | +| penalty/lcb_mean | 1.73 | +| penalty/lcb_std | 2.76 | +| rollout_info/dynamics_penalty_mean | 0.000259 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1000000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.8921 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.9854 +num rollout transitions: 250000, reward mean: 8.9102 +num rollout transitions: 250000, reward mean: 8.9302 +num rollout transitions: 250000, reward mean: 8.9417 +num rollout transitions: 250000, reward mean: 8.9156 +num rollout transitions: 250000, reward mean: 8.9326 +num rollout transitions: 250000, reward mean: 8.9816 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9902 +num rollout transitions: 250000, reward mean: 8.9155 +num rollout transitions: 250000, reward mean: 8.9241 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9667 +num rollout transitions: 250000, reward mean: 8.9637 +num rollout transitions: 250000, reward mean: 9.0095 +num rollout transitions: 250000, reward mean: 8.9300 +num rollout transitions: 250000, reward mean: 8.9143 +num rollout transitions: 250000, reward mean: 8.9302 +num rollout transitions: 250000, reward mean: 8.9201 +num rollout transitions: 250000, reward mean: 8.9607 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9685 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9238 +num rollout transitions: 250000, reward mean: 8.9681 +num rollout transitions: 250000, reward mean: 8.9439 +num rollout transitions: 250000, reward mean: 8.9677 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9144 +num rollout transitions: 250000, reward mean: 8.8737 +num rollout transitions: 250000, reward mean: 8.9712 +num rollout transitions: 250000, reward mean: 8.9191 +num rollout transitions: 250000, reward mean: 8.9583 +num rollout transitions: 250000, reward mean: 8.9403 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9354 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9597 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9525 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9421 +num rollout transitions: 250000, reward mean: 8.9501 +---------------------------------------------------------------------------------- +| alpha | 0.376 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 89.8 | +| eval/normalized_episode_reward_std | 3.44 | +| loss/actor | -963 | +| loss/alpha | 0.00198 | +| loss/critic | 86.9 | +| penalty/lcb_mean | 1.74 | +| penalty/lcb_std | 2.81 | +| rollout_info/dynamics_penalty_mean | 0.000275 | +| rollout_info/dynamics_penalty_std | 0.0159 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9075 +num rollout transitions: 250000, reward mean: 8.9477 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9449 +num rollout transitions: 250000, reward mean: 8.9380 +num rollout transitions: 250000, reward mean: 8.9355 +num rollout transitions: 250000, reward mean: 8.9336 +num rollout transitions: 250000, reward mean: 8.9221 +num rollout transitions: 250000, reward mean: 8.9774 +num rollout transitions: 250000, reward mean: 8.9324 +num rollout transitions: 250000, reward mean: 8.9618 +num rollout transitions: 250000, reward mean: 8.9659 +num rollout transitions: 250000, reward mean: 8.9626 +num rollout transitions: 250000, reward mean: 8.9301 +num rollout transitions: 250000, reward mean: 8.9170 +num rollout transitions: 250000, reward mean: 8.9513 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9662 +num rollout transitions: 250000, reward mean: 8.9324 +num rollout transitions: 250000, reward mean: 8.8836 +num rollout transitions: 250000, reward mean: 8.9212 +num rollout transitions: 250000, reward mean: 8.9153 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9432 +num rollout transitions: 250000, reward mean: 8.9676 +num rollout transitions: 250000, reward mean: 8.9251 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9312 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9268 +num rollout transitions: 250000, reward mean: 8.9600 +num rollout transitions: 250000, reward mean: 8.9566 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9277 +num rollout transitions: 250000, reward mean: 8.9514 +num rollout transitions: 250000, reward mean: 8.9549 +num rollout transitions: 250000, reward mean: 8.9226 +num rollout transitions: 250000, reward mean: 8.9734 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9034 +num rollout transitions: 250000, reward mean: 8.9658 +num rollout transitions: 250000, reward mean: 8.9304 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9221 +num rollout transitions: 250000, reward mean: 8.9392 +num rollout transitions: 250000, reward mean: 8.9588 +num rollout transitions: 250000, reward mean: 8.9494 +---------------------------------------------------------------------------------- +| alpha | 0.372 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 100 | +| eval/normalized_episode_reward_std | 7.62 | +| loss/actor | -967 | +| loss/alpha | -0.00107 | +| loss/critic | 87.4 | +| penalty/lcb_mean | 1.73 | +| penalty/lcb_std | 2.81 | +| rollout_info/dynamics_penalty_mean | 0.000251 | +| rollout_info/dynamics_penalty_std | 0.0152 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.9510 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9639 +num rollout transitions: 250000, reward mean: 8.9206 +num rollout transitions: 250000, reward mean: 8.9338 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9325 +num rollout transitions: 250000, reward mean: 8.9513 +num rollout transitions: 250000, reward mean: 8.9047 +num rollout transitions: 250000, reward mean: 8.9720 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9420 +num rollout transitions: 250000, reward mean: 8.9618 +num rollout transitions: 250000, reward mean: 8.9373 +num rollout transitions: 250000, reward mean: 8.9296 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9337 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9388 +num rollout transitions: 250000, reward mean: 8.9845 +num rollout transitions: 250000, reward mean: 8.9468 +num rollout transitions: 250000, reward mean: 8.9161 +num rollout transitions: 250000, reward mean: 8.9575 +num rollout transitions: 250000, reward mean: 8.9374 +num rollout transitions: 250000, reward mean: 8.9493 +num rollout transitions: 250000, reward mean: 8.9352 +num rollout transitions: 250000, reward mean: 8.9652 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9388 +num rollout transitions: 250000, reward mean: 8.9393 +num rollout transitions: 250000, reward mean: 8.9427 +num rollout transitions: 250000, reward mean: 8.9677 +num rollout transitions: 250000, reward mean: 8.9416 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9562 +num rollout transitions: 250000, reward mean: 8.9254 +num rollout transitions: 250000, reward mean: 8.9606 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9258 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.9606 +num rollout transitions: 250000, reward mean: 8.9630 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9196 +num rollout transitions: 250000, reward mean: 8.9815 +num rollout transitions: 250000, reward mean: 8.9518 +---------------------------------------------------------------------------------- +| alpha | 0.372 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 95.9 | +| eval/normalized_episode_reward_std | 16.3 | +| loss/actor | -968 | +| loss/alpha | 0.000443 | +| loss/critic | 85.7 | +| penalty/lcb_mean | 1.71 | +| penalty/lcb_std | 2.8 | +| rollout_info/dynamics_penalty_mean | 0.000262 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9384 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9502 +num rollout transitions: 250000, reward mean: 8.9249 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.9683 +num rollout transitions: 250000, reward mean: 8.9566 +num rollout transitions: 250000, reward mean: 8.9582 +num rollout transitions: 250000, reward mean: 8.9333 +num rollout transitions: 250000, reward mean: 8.9497 +num rollout transitions: 250000, reward mean: 8.9378 +num rollout transitions: 250000, reward mean: 8.9376 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9640 +num rollout transitions: 250000, reward mean: 8.9571 +num rollout transitions: 250000, reward mean: 8.9670 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9782 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9241 +num rollout transitions: 250000, reward mean: 8.9187 +num rollout transitions: 250000, reward mean: 8.9255 +num rollout transitions: 250000, reward mean: 8.9180 +num rollout transitions: 250000, reward mean: 8.9678 +num rollout transitions: 250000, reward mean: 8.9475 +num rollout transitions: 250000, reward mean: 8.9855 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9517 +num rollout transitions: 250000, reward mean: 8.9183 +num rollout transitions: 250000, reward mean: 8.9367 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.9673 +num rollout transitions: 250000, reward mean: 8.9675 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9676 +num rollout transitions: 250000, reward mean: 8.9213 +num rollout transitions: 250000, reward mean: 8.9237 +num rollout transitions: 250000, reward mean: 8.9315 +num rollout transitions: 250000, reward mean: 8.9401 +num rollout transitions: 250000, reward mean: 8.9320 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9659 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9785 +num rollout transitions: 250000, reward mean: 8.9432 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9701 +num rollout transitions: 250000, reward mean: 8.9503 +num rollout transitions: 250000, reward mean: 8.9331 +---------------------------------------------------------------------------------- +| alpha | 0.376 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.5 | +| eval/normalized_episode_reward_std | 6.21 | +| loss/actor | -970 | +| loss/alpha | 0.000396 | +| loss/critic | 85.1 | +| penalty/lcb_mean | 1.72 | +| penalty/lcb_std | 2.8 | +| rollout_info/dynamics_penalty_mean | 0.000258 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9369 +num rollout transitions: 250000, reward mean: 8.9533 +num rollout transitions: 250000, reward mean: 8.9489 +num rollout transitions: 250000, reward mean: 8.9059 +num rollout transitions: 250000, reward mean: 8.9697 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9742 +num rollout transitions: 250000, reward mean: 8.9502 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9566 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9305 +num rollout transitions: 250000, reward mean: 8.9955 +num rollout transitions: 250000, reward mean: 8.9543 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9200 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9708 +num rollout transitions: 250000, reward mean: 8.9467 +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.9644 +num rollout transitions: 250000, reward mean: 8.9705 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9439 +num rollout transitions: 250000, reward mean: 8.9231 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9283 +num rollout transitions: 250000, reward mean: 8.9664 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9730 +num rollout transitions: 250000, reward mean: 8.9729 +num rollout transitions: 250000, reward mean: 8.9436 +num rollout transitions: 250000, reward mean: 8.9548 +num rollout transitions: 250000, reward mean: 8.9586 +num rollout transitions: 250000, reward mean: 8.9597 +num rollout transitions: 250000, reward mean: 8.9171 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9116 +num rollout transitions: 250000, reward mean: 8.9347 +num rollout transitions: 250000, reward mean: 8.9483 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9389 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9655 +num rollout transitions: 250000, reward mean: 8.9144 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.9577 +num rollout transitions: 250000, reward mean: 8.9664 +num rollout transitions: 250000, reward mean: 8.9430 +----------------------------------------------------------------------------------- +| alpha | 0.375 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 89.5 | +| eval/normalized_episode_reward_std | 3.18 | +| loss/actor | -969 | +| loss/alpha | -0.000823 | +| loss/critic | 88.3 | +| penalty/lcb_mean | 1.73 | +| penalty/lcb_std | 2.84 | +| rollout_info/dynamics_penalty_mean | 0.000262 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1250000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.8975 +num rollout transitions: 250000, reward mean: 8.9250 +num rollout transitions: 250000, reward mean: 8.9252 +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9950 +num rollout transitions: 250000, reward mean: 8.9119 +num rollout transitions: 250000, reward mean: 8.9392 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9042 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9688 +num rollout transitions: 250000, reward mean: 8.9884 +num rollout transitions: 250000, reward mean: 8.9601 +num rollout transitions: 250000, reward mean: 8.9353 +num rollout transitions: 250000, reward mean: 8.9253 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9643 +num rollout transitions: 250000, reward mean: 8.9573 +num rollout transitions: 250000, reward mean: 8.9822 +num rollout transitions: 250000, reward mean: 8.9337 +num rollout transitions: 250000, reward mean: 8.9368 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9393 +num rollout transitions: 250000, reward mean: 8.9624 +num rollout transitions: 250000, reward mean: 8.9012 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9693 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9455 +num rollout transitions: 250000, reward mean: 8.9480 +num rollout transitions: 250000, reward mean: 8.9496 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.9431 +num rollout transitions: 250000, reward mean: 8.9923 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9610 +num rollout transitions: 250000, reward mean: 8.9170 +num rollout transitions: 250000, reward mean: 8.9364 +num rollout transitions: 250000, reward mean: 8.9244 +num rollout transitions: 250000, reward mean: 8.9660 +num rollout transitions: 250000, reward mean: 8.9518 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9314 +num rollout transitions: 250000, reward mean: 8.9129 +num rollout transitions: 250000, reward mean: 8.9522 +num rollout transitions: 250000, reward mean: 8.9627 +num rollout transitions: 250000, reward mean: 8.9502 +---------------------------------------------------------------------------------- +| alpha | 0.367 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 87.3 | +| eval/normalized_episode_reward_std | 20.7 | +| loss/actor | -970 | +| loss/alpha | -0.00173 | +| loss/critic | 87.2 | +| penalty/lcb_mean | 1.71 | +| penalty/lcb_std | 2.83 | +| rollout_info/dynamics_penalty_mean | 0.000266 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9469 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9261 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9713 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9632 +num rollout transitions: 250000, reward mean: 8.9505 +num rollout transitions: 250000, reward mean: 8.9520 +num rollout transitions: 250000, reward mean: 8.9285 +num rollout transitions: 250000, reward mean: 8.9975 +num rollout transitions: 250000, reward mean: 8.9365 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9564 +num rollout transitions: 250000, reward mean: 8.9245 +num rollout transitions: 250000, reward mean: 8.9642 +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.9396 +num rollout transitions: 250000, reward mean: 8.9453 +num rollout transitions: 250000, reward mean: 8.9505 +num rollout transitions: 250000, reward mean: 8.9272 +num rollout transitions: 250000, reward mean: 8.9588 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 8.9422 +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9622 +num rollout transitions: 250000, reward mean: 8.9129 +num rollout transitions: 250000, reward mean: 8.9416 +num rollout transitions: 250000, reward mean: 8.9709 +num rollout transitions: 250000, reward mean: 8.9231 +num rollout transitions: 250000, reward mean: 8.9630 +num rollout transitions: 250000, reward mean: 9.0019 +num rollout transitions: 250000, reward mean: 8.9573 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9133 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9858 +num rollout transitions: 250000, reward mean: 8.9205 +num rollout transitions: 250000, reward mean: 8.9062 +num rollout transitions: 250000, reward mean: 8.8991 +num rollout transitions: 250000, reward mean: 8.9275 +num rollout transitions: 250000, reward mean: 8.9698 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9428 +num rollout transitions: 250000, reward mean: 8.9161 +num rollout transitions: 250000, reward mean: 8.9738 +num rollout transitions: 250000, reward mean: 8.9668 +---------------------------------------------------------------------------------- +| alpha | 0.37 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 84.5 | +| eval/normalized_episode_reward_std | 3.36 | +| loss/actor | -969 | +| loss/alpha | 0.000126 | +| loss/critic | 92.6 | +| penalty/lcb_mean | 1.74 | +| penalty/lcb_std | 2.93 | +| rollout_info/dynamics_penalty_mean | 0.000282 | +| rollout_info/dynamics_penalty_std | 0.0161 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.8851 +num rollout transitions: 250000, reward mean: 8.9373 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9677 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9714 +num rollout transitions: 250000, reward mean: 8.9254 +num rollout transitions: 250000, reward mean: 8.9147 +num rollout transitions: 250000, reward mean: 8.9672 +num rollout transitions: 250000, reward mean: 8.9691 +num rollout transitions: 250000, reward mean: 8.9588 +num rollout transitions: 250000, reward mean: 8.9258 +num rollout transitions: 250000, reward mean: 8.9592 +num rollout transitions: 250000, reward mean: 8.9276 +num rollout transitions: 250000, reward mean: 8.9445 +num rollout transitions: 250000, reward mean: 8.9532 +num rollout transitions: 250000, reward mean: 8.9077 +num rollout transitions: 250000, reward mean: 8.9384 +num rollout transitions: 250000, reward mean: 8.9507 +num rollout transitions: 250000, reward mean: 8.9538 +num rollout transitions: 250000, reward mean: 8.9417 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9242 +num rollout transitions: 250000, reward mean: 8.9452 +num rollout transitions: 250000, reward mean: 8.9628 +num rollout transitions: 250000, reward mean: 8.9421 +num rollout transitions: 250000, reward mean: 8.9609 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9521 +num rollout transitions: 250000, reward mean: 8.9483 +num rollout transitions: 250000, reward mean: 8.9837 +num rollout transitions: 250000, reward mean: 8.9426 +num rollout transitions: 250000, reward mean: 8.9489 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9464 +num rollout transitions: 250000, reward mean: 8.9136 +num rollout transitions: 250000, reward mean: 8.9429 +num rollout transitions: 250000, reward mean: 8.9880 +num rollout transitions: 250000, reward mean: 8.9577 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9262 +num rollout transitions: 250000, reward mean: 8.9532 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9066 +num rollout transitions: 250000, reward mean: 8.9261 +num rollout transitions: 250000, reward mean: 8.9228 +num rollout transitions: 250000, reward mean: 8.9484 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9429 +---------------------------------------------------------------------------------- +| alpha | 0.369 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.4 | +| eval/normalized_episode_reward_std | 3.29 | +| loss/actor | -974 | +| loss/alpha | 0.00261 | +| loss/critic | 81.2 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.74 | +| rollout_info/dynamics_penalty_mean | 0.000262 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9557 +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.9784 +num rollout transitions: 250000, reward mean: 8.9525 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9143 +num rollout transitions: 250000, reward mean: 8.9517 +num rollout transitions: 250000, reward mean: 8.9693 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9705 +num rollout transitions: 250000, reward mean: 8.9019 +num rollout transitions: 250000, reward mean: 8.9614 +num rollout transitions: 250000, reward mean: 8.9503 +num rollout transitions: 250000, reward mean: 8.9756 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.8834 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9531 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9028 +num rollout transitions: 250000, reward mean: 8.9151 +num rollout transitions: 250000, reward mean: 8.9945 +num rollout transitions: 250000, reward mean: 8.9077 +num rollout transitions: 250000, reward mean: 8.9173 +num rollout transitions: 250000, reward mean: 8.9414 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9119 +num rollout transitions: 250000, reward mean: 8.9266 +num rollout transitions: 250000, reward mean: 8.9715 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9414 +num rollout transitions: 250000, reward mean: 8.9442 +num rollout transitions: 250000, reward mean: 8.9260 +num rollout transitions: 250000, reward mean: 8.9531 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9848 +num rollout transitions: 250000, reward mean: 8.9464 +num rollout transitions: 250000, reward mean: 8.9323 +num rollout transitions: 250000, reward mean: 8.9302 +num rollout transitions: 250000, reward mean: 8.9497 +num rollout transitions: 250000, reward mean: 8.9510 +---------------------------------------------------------------------------------- +| alpha | 0.367 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 91.8 | +| eval/normalized_episode_reward_std | 22.3 | +| loss/actor | -975 | +| loss/alpha | -0.00137 | +| loss/critic | 88.4 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.88 | +| rollout_info/dynamics_penalty_mean | 0.000266 | +| rollout_info/dynamics_penalty_std | 0.0157 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9588 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9693 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9203 +num rollout transitions: 250000, reward mean: 8.9601 +num rollout transitions: 250000, reward mean: 8.9336 +num rollout transitions: 250000, reward mean: 8.9301 +num rollout transitions: 250000, reward mean: 8.9326 +num rollout transitions: 250000, reward mean: 8.9660 +num rollout transitions: 250000, reward mean: 8.9392 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9159 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9122 +num rollout transitions: 250000, reward mean: 8.9239 +num rollout transitions: 250000, reward mean: 8.9535 +num rollout transitions: 250000, reward mean: 8.9745 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9715 +num rollout transitions: 250000, reward mean: 8.8995 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9723 +num rollout transitions: 250000, reward mean: 8.9466 +num rollout transitions: 250000, reward mean: 8.9551 +num rollout transitions: 250000, reward mean: 8.9121 +num rollout transitions: 250000, reward mean: 8.9497 +num rollout transitions: 250000, reward mean: 8.9211 +num rollout transitions: 250000, reward mean: 8.9177 +num rollout transitions: 250000, reward mean: 8.9366 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9314 +num rollout transitions: 250000, reward mean: 8.9610 +num rollout transitions: 250000, reward mean: 8.9259 +num rollout transitions: 250000, reward mean: 8.9480 +num rollout transitions: 250000, reward mean: 8.9709 +num rollout transitions: 250000, reward mean: 8.9309 +num rollout transitions: 250000, reward mean: 8.9110 +num rollout transitions: 250000, reward mean: 8.9624 +num rollout transitions: 250000, reward mean: 8.9427 +num rollout transitions: 250000, reward mean: 8.9568 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9365 +num rollout transitions: 250000, reward mean: 8.9229 +---------------------------------------------------------------------------------- +| alpha | 0.366 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 3.46 | +| loss/actor | -977 | +| loss/alpha | 0.000148 | +| loss/critic | 84.5 | +| penalty/lcb_mean | 1.69 | +| penalty/lcb_std | 2.81 | +| rollout_info/dynamics_penalty_mean | 0.000263 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9733 +num rollout transitions: 250000, reward mean: 8.9173 +num rollout transitions: 250000, reward mean: 8.9680 +num rollout transitions: 250000, reward mean: 8.9514 +num rollout transitions: 250000, reward mean: 8.9329 +num rollout transitions: 250000, reward mean: 8.9746 +num rollout transitions: 250000, reward mean: 8.9256 +num rollout transitions: 250000, reward mean: 8.9533 +num rollout transitions: 250000, reward mean: 8.9690 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9588 +num rollout transitions: 250000, reward mean: 8.9376 +num rollout transitions: 250000, reward mean: 8.9083 +num rollout transitions: 250000, reward mean: 8.9714 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9673 +num rollout transitions: 250000, reward mean: 8.9469 +num rollout transitions: 250000, reward mean: 8.9702 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9514 +num rollout transitions: 250000, reward mean: 8.9467 +num rollout transitions: 250000, reward mean: 8.9300 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9587 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9244 +num rollout transitions: 250000, reward mean: 8.9262 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.9113 +num rollout transitions: 250000, reward mean: 8.9320 +num rollout transitions: 250000, reward mean: 8.9511 +num rollout transitions: 250000, reward mean: 8.9551 +num rollout transitions: 250000, reward mean: 8.9428 +num rollout transitions: 250000, reward mean: 8.9152 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9618 +num rollout transitions: 250000, reward mean: 8.9679 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9472 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9636 +num rollout transitions: 250000, reward mean: 8.9098 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9996 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9648 +----------------------------------------------------------------------------------- +| alpha | 0.361 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 2.89 | +| loss/actor | -980 | +| loss/alpha | -0.000994 | +| loss/critic | 79.7 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.72 | +| rollout_info/dynamics_penalty_mean | 0.000289 | +| rollout_info/dynamics_penalty_std | 0.0163 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1550000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9698 +num rollout transitions: 250000, reward mean: 8.9657 +num rollout transitions: 250000, reward mean: 8.9344 +num rollout transitions: 250000, reward mean: 8.9544 +num rollout transitions: 250000, reward mean: 8.9374 +num rollout transitions: 250000, reward mean: 8.9639 +num rollout transitions: 250000, reward mean: 8.9873 +num rollout transitions: 250000, reward mean: 8.9141 +num rollout transitions: 250000, reward mean: 8.9706 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.9619 +num rollout transitions: 250000, reward mean: 8.9614 +num rollout transitions: 250000, reward mean: 8.9422 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9428 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9843 +num rollout transitions: 250000, reward mean: 8.9625 +num rollout transitions: 250000, reward mean: 8.9497 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9549 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9764 +num rollout transitions: 250000, reward mean: 8.9509 +num rollout transitions: 250000, reward mean: 8.9558 +num rollout transitions: 250000, reward mean: 8.9331 +num rollout transitions: 250000, reward mean: 8.9520 +num rollout transitions: 250000, reward mean: 8.9160 +num rollout transitions: 250000, reward mean: 8.9186 +num rollout transitions: 250000, reward mean: 8.9569 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9753 +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.9287 +num rollout transitions: 250000, reward mean: 8.9449 +num rollout transitions: 250000, reward mean: 8.9575 +num rollout transitions: 250000, reward mean: 8.9273 +num rollout transitions: 250000, reward mean: 8.9332 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9699 +num rollout transitions: 250000, reward mean: 8.9677 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9325 +----------------------------------------------------------------------------------- +| alpha | 0.36 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 3.25 | +| loss/actor | -982 | +| loss/alpha | -0.000699 | +| loss/critic | 80 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.73 | +| rollout_info/dynamics_penalty_mean | 0.00027 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1600000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9474 +num rollout transitions: 250000, reward mean: 8.9428 +num rollout transitions: 250000, reward mean: 8.9366 +num rollout transitions: 250000, reward mean: 8.9683 +num rollout transitions: 250000, reward mean: 8.9394 +num rollout transitions: 250000, reward mean: 8.9314 +num rollout transitions: 250000, reward mean: 8.9618 +num rollout transitions: 250000, reward mean: 8.9650 +num rollout transitions: 250000, reward mean: 8.9298 +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.9746 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9814 +num rollout transitions: 250000, reward mean: 8.9652 +num rollout transitions: 250000, reward mean: 8.9381 +num rollout transitions: 250000, reward mean: 8.9463 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9716 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9795 +num rollout transitions: 250000, reward mean: 8.9116 +num rollout transitions: 250000, reward mean: 8.9626 +num rollout transitions: 250000, reward mean: 8.9428 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9236 +num rollout transitions: 250000, reward mean: 8.9508 +num rollout transitions: 250000, reward mean: 8.9667 +num rollout transitions: 250000, reward mean: 8.9244 +num rollout transitions: 250000, reward mean: 8.9278 +num rollout transitions: 250000, reward mean: 8.9390 +num rollout transitions: 250000, reward mean: 8.9392 +num rollout transitions: 250000, reward mean: 8.9416 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9662 +num rollout transitions: 250000, reward mean: 8.9651 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9591 +num rollout transitions: 250000, reward mean: 8.9564 +num rollout transitions: 250000, reward mean: 8.9639 +num rollout transitions: 250000, reward mean: 8.9611 +num rollout transitions: 250000, reward mean: 8.9268 +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.9612 +num rollout transitions: 250000, reward mean: 8.9614 +num rollout transitions: 250000, reward mean: 8.9668 +num rollout transitions: 250000, reward mean: 8.9376 +---------------------------------------------------------------------------------- +| alpha | 0.357 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.1 | +| eval/normalized_episode_reward_std | 14.3 | +| loss/actor | -982 | +| loss/alpha | 5.04e-05 | +| loss/critic | 78.8 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.72 | +| rollout_info/dynamics_penalty_mean | 0.000273 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9469 +num rollout transitions: 250000, reward mean: 8.9476 +num rollout transitions: 250000, reward mean: 8.9235 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9159 +num rollout transitions: 250000, reward mean: 8.9573 +num rollout transitions: 250000, reward mean: 8.9815 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9062 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9749 +num rollout transitions: 250000, reward mean: 8.9387 +num rollout transitions: 250000, reward mean: 8.9484 +num rollout transitions: 250000, reward mean: 8.9568 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9275 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9832 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9667 +num rollout transitions: 250000, reward mean: 8.9270 +num rollout transitions: 250000, reward mean: 8.9152 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9686 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9749 +num rollout transitions: 250000, reward mean: 8.9610 +num rollout transitions: 250000, reward mean: 8.9436 +num rollout transitions: 250000, reward mean: 8.9234 +num rollout transitions: 250000, reward mean: 8.9798 +num rollout transitions: 250000, reward mean: 8.9521 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9543 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.8894 +num rollout transitions: 250000, reward mean: 9.0012 +num rollout transitions: 250000, reward mean: 8.9410 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9560 +num rollout transitions: 250000, reward mean: 8.9140 +num rollout transitions: 250000, reward mean: 8.9110 +num rollout transitions: 250000, reward mean: 8.9169 +num rollout transitions: 250000, reward mean: 8.9337 +num rollout transitions: 250000, reward mean: 8.9466 +num rollout transitions: 250000, reward mean: 8.9493 +----------------------------------------------------------------------------------- +| alpha | 0.355 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 98.5 | +| eval/normalized_episode_reward_std | 3.07 | +| loss/actor | -985 | +| loss/alpha | -0.000558 | +| loss/critic | 82.9 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.81 | +| rollout_info/dynamics_penalty_mean | 0.000249 | +| rollout_info/dynamics_penalty_std | 0.0152 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1700000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9506 +num rollout transitions: 250000, reward mean: 8.9713 +num rollout transitions: 250000, reward mean: 8.9373 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9590 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9763 +num rollout transitions: 250000, reward mean: 8.9472 +num rollout transitions: 250000, reward mean: 8.9541 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9606 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9595 +num rollout transitions: 250000, reward mean: 8.9863 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9680 +num rollout transitions: 250000, reward mean: 8.9792 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9223 +num rollout transitions: 250000, reward mean: 8.9264 +num rollout transitions: 250000, reward mean: 9.0065 +num rollout transitions: 250000, reward mean: 8.9215 +num rollout transitions: 250000, reward mean: 8.9400 +num rollout transitions: 250000, reward mean: 8.9464 +num rollout transitions: 250000, reward mean: 8.9294 +num rollout transitions: 250000, reward mean: 8.9104 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9268 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9525 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9514 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9467 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9315 +num rollout transitions: 250000, reward mean: 8.9682 +num rollout transitions: 250000, reward mean: 8.9250 +num rollout transitions: 250000, reward mean: 8.9656 +num rollout transitions: 250000, reward mean: 8.9112 +num rollout transitions: 250000, reward mean: 8.9672 +num rollout transitions: 250000, reward mean: 8.9315 +num rollout transitions: 250000, reward mean: 8.9760 +num rollout transitions: 250000, reward mean: 8.9650 +num rollout transitions: 250000, reward mean: 8.9207 +num rollout transitions: 250000, reward mean: 8.9215 +---------------------------------------------------------------------------------- +| alpha | 0.357 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 7.94 | +| loss/actor | -981 | +| loss/alpha | 0.00301 | +| loss/critic | 107 | +| penalty/lcb_mean | 1.73 | +| penalty/lcb_std | 3.25 | +| rollout_info/dynamics_penalty_mean | 0.000269 | +| rollout_info/dynamics_penalty_std | 0.0157 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9378 +num rollout transitions: 250000, reward mean: 8.9187 +num rollout transitions: 250000, reward mean: 8.9459 +num rollout transitions: 250000, reward mean: 8.9157 +num rollout transitions: 250000, reward mean: 8.9371 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9273 +num rollout transitions: 250000, reward mean: 8.9826 +num rollout transitions: 250000, reward mean: 8.9421 +num rollout transitions: 250000, reward mean: 8.9243 +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.9237 +num rollout transitions: 250000, reward mean: 8.9376 +num rollout transitions: 250000, reward mean: 8.9616 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.9395 +num rollout transitions: 250000, reward mean: 8.9457 +num rollout transitions: 250000, reward mean: 8.9315 +num rollout transitions: 250000, reward mean: 8.9369 +num rollout transitions: 250000, reward mean: 8.9786 +num rollout transitions: 250000, reward mean: 8.9097 +num rollout transitions: 250000, reward mean: 8.9617 +num rollout transitions: 250000, reward mean: 8.9766 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9513 +num rollout transitions: 250000, reward mean: 8.9037 +num rollout transitions: 250000, reward mean: 8.9682 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9285 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9285 +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9727 +num rollout transitions: 250000, reward mean: 8.9517 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9819 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9380 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9788 +num rollout transitions: 250000, reward mean: 8.9630 +num rollout transitions: 250000, reward mean: 8.9688 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9044 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9561 +---------------------------------------------------------------------------------- +| alpha | 0.353 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 96.4 | +| eval/normalized_episode_reward_std | 14.7 | +| loss/actor | -982 | +| loss/alpha | -0.00367 | +| loss/critic | 79.2 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.74 | +| rollout_info/dynamics_penalty_mean | 0.000253 | +| rollout_info/dynamics_penalty_std | 0.0152 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9475 +num rollout transitions: 250000, reward mean: 8.9543 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9569 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9177 +num rollout transitions: 250000, reward mean: 8.9711 +num rollout transitions: 250000, reward mean: 8.9263 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.9658 +num rollout transitions: 250000, reward mean: 8.9380 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9752 +num rollout transitions: 250000, reward mean: 8.9389 +num rollout transitions: 250000, reward mean: 8.9054 +num rollout transitions: 250000, reward mean: 8.9296 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9002 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.9403 +num rollout transitions: 250000, reward mean: 8.9595 +num rollout transitions: 250000, reward mean: 8.9703 +num rollout transitions: 250000, reward mean: 8.9885 +num rollout transitions: 250000, reward mean: 8.9582 +num rollout transitions: 250000, reward mean: 8.9649 +num rollout transitions: 250000, reward mean: 8.9263 +num rollout transitions: 250000, reward mean: 8.9305 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9834 +num rollout transitions: 250000, reward mean: 8.9578 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9873 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9298 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9872 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.9213 +num rollout transitions: 250000, reward mean: 8.9510 +num rollout transitions: 250000, reward mean: 8.9788 +num rollout transitions: 250000, reward mean: 8.9645 +num rollout transitions: 250000, reward mean: 8.9682 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9323 +num rollout transitions: 250000, reward mean: 8.9707 +num rollout transitions: 250000, reward mean: 8.9355 +num rollout transitions: 250000, reward mean: 8.9559 +---------------------------------------------------------------------------------- +| alpha | 0.353 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 10.6 | +| loss/actor | -987 | +| loss/alpha | 0.00117 | +| loss/critic | 78.8 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.73 | +| rollout_info/dynamics_penalty_mean | 0.000247 | +| rollout_info/dynamics_penalty_std | 0.015 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9506 +num rollout transitions: 250000, reward mean: 8.9579 +num rollout transitions: 250000, reward mean: 8.9226 +num rollout transitions: 250000, reward mean: 8.9572 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.9646 +num rollout transitions: 250000, reward mean: 8.9579 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9316 +num rollout transitions: 250000, reward mean: 8.9358 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9767 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9556 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9522 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9573 +num rollout transitions: 250000, reward mean: 8.9115 +num rollout transitions: 250000, reward mean: 8.9850 +num rollout transitions: 250000, reward mean: 8.9171 +num rollout transitions: 250000, reward mean: 8.9052 +num rollout transitions: 250000, reward mean: 8.9302 +num rollout transitions: 250000, reward mean: 8.9128 +num rollout transitions: 250000, reward mean: 8.9213 +num rollout transitions: 250000, reward mean: 8.9313 +num rollout transitions: 250000, reward mean: 8.9508 +num rollout transitions: 250000, reward mean: 8.9820 +num rollout transitions: 250000, reward mean: 8.9775 +num rollout transitions: 250000, reward mean: 8.9273 +num rollout transitions: 250000, reward mean: 8.9445 +num rollout transitions: 250000, reward mean: 8.9510 +num rollout transitions: 250000, reward mean: 8.9598 +num rollout transitions: 250000, reward mean: 8.9835 +num rollout transitions: 250000, reward mean: 8.9586 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9396 +num rollout transitions: 250000, reward mean: 8.9413 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9452 +num rollout transitions: 250000, reward mean: 8.9556 +num rollout transitions: 250000, reward mean: 8.9757 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9247 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9527 +num rollout transitions: 250000, reward mean: 8.9519 +----------------------------------------------------------------------------------- +| alpha | 0.351 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 5.99 | +| loss/actor | -988 | +| loss/alpha | -0.000881 | +| loss/critic | 80.4 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.79 | +| rollout_info/dynamics_penalty_mean | 0.000259 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1900000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9637 +num rollout transitions: 250000, reward mean: 8.9767 +num rollout transitions: 250000, reward mean: 8.9614 +num rollout transitions: 250000, reward mean: 8.9438 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9933 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9713 +num rollout transitions: 250000, reward mean: 8.9686 +num rollout transitions: 250000, reward mean: 8.9236 +num rollout transitions: 250000, reward mean: 8.9575 +num rollout transitions: 250000, reward mean: 8.9841 +num rollout transitions: 250000, reward mean: 8.9295 +num rollout transitions: 250000, reward mean: 8.9355 +num rollout transitions: 250000, reward mean: 8.9789 +num rollout transitions: 250000, reward mean: 8.9432 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.9537 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.9726 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9295 +num rollout transitions: 250000, reward mean: 8.9461 +num rollout transitions: 250000, reward mean: 8.9270 +num rollout transitions: 250000, reward mean: 8.9452 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9436 +num rollout transitions: 250000, reward mean: 8.9866 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9236 +num rollout transitions: 250000, reward mean: 8.9621 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9509 +num rollout transitions: 250000, reward mean: 8.9296 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9897 +num rollout transitions: 250000, reward mean: 8.9719 +num rollout transitions: 250000, reward mean: 8.9475 +num rollout transitions: 250000, reward mean: 8.9498 +num rollout transitions: 250000, reward mean: 8.9504 +num rollout transitions: 250000, reward mean: 8.9741 +num rollout transitions: 250000, reward mean: 8.9496 +num rollout transitions: 250000, reward mean: 8.9835 +num rollout transitions: 250000, reward mean: 8.9503 +num rollout transitions: 250000, reward mean: 8.9495 +---------------------------------------------------------------------------------- +| alpha | 0.348 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.1 | +| eval/normalized_episode_reward_std | 3.35 | +| loss/actor | -988 | +| loss/alpha | 0.000852 | +| loss/critic | 80.9 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.79 | +| rollout_info/dynamics_penalty_mean | 0.000262 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9403 +num rollout transitions: 250000, reward mean: 8.9522 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9680 +num rollout transitions: 250000, reward mean: 8.9733 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9474 +num rollout transitions: 250000, reward mean: 8.9456 +num rollout transitions: 250000, reward mean: 8.9461 +num rollout transitions: 250000, reward mean: 8.9863 +num rollout transitions: 250000, reward mean: 8.9717 +num rollout transitions: 250000, reward mean: 8.9469 +num rollout transitions: 250000, reward mean: 8.9236 +num rollout transitions: 250000, reward mean: 8.9438 +num rollout transitions: 250000, reward mean: 8.9243 +num rollout transitions: 250000, reward mean: 8.9360 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9573 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9122 +num rollout transitions: 250000, reward mean: 8.9427 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9501 +num rollout transitions: 250000, reward mean: 8.9706 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9120 +num rollout transitions: 250000, reward mean: 8.9537 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9287 +num rollout transitions: 250000, reward mean: 8.9583 +num rollout transitions: 250000, reward mean: 8.9472 +num rollout transitions: 250000, reward mean: 8.9847 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.9759 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9463 +num rollout transitions: 250000, reward mean: 8.9527 +num rollout transitions: 250000, reward mean: 8.9336 +num rollout transitions: 250000, reward mean: 8.9617 +num rollout transitions: 250000, reward mean: 8.9006 +num rollout transitions: 250000, reward mean: 8.9266 +num rollout transitions: 250000, reward mean: 8.9734 +num rollout transitions: 250000, reward mean: 8.9510 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9750 +---------------------------------------------------------------------------------- +| alpha | 0.352 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 3.16 | +| loss/actor | -989 | +| loss/alpha | 0.000538 | +| loss/critic | 81.7 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.81 | +| rollout_info/dynamics_penalty_mean | 0.000279 | +| rollout_info/dynamics_penalty_std | 0.0161 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9502 +num rollout transitions: 250000, reward mean: 8.8907 +num rollout transitions: 250000, reward mean: 8.9496 +num rollout transitions: 250000, reward mean: 8.9581 +num rollout transitions: 250000, reward mean: 8.9875 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9805 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9459 +num rollout transitions: 250000, reward mean: 8.9278 +num rollout transitions: 250000, reward mean: 8.9602 +num rollout transitions: 250000, reward mean: 8.9676 +num rollout transitions: 250000, reward mean: 8.9661 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9610 +num rollout transitions: 250000, reward mean: 8.9737 +num rollout transitions: 250000, reward mean: 8.9469 +num rollout transitions: 250000, reward mean: 8.9800 +num rollout transitions: 250000, reward mean: 8.9507 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9647 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9522 +num rollout transitions: 250000, reward mean: 8.9729 +num rollout transitions: 250000, reward mean: 8.9345 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9532 +num rollout transitions: 250000, reward mean: 8.9878 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9738 +num rollout transitions: 250000, reward mean: 8.9700 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9709 +num rollout transitions: 250000, reward mean: 8.9354 +num rollout transitions: 250000, reward mean: 8.9741 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9644 +num rollout transitions: 250000, reward mean: 8.9718 +num rollout transitions: 250000, reward mean: 8.9809 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9452 +num rollout transitions: 250000, reward mean: 8.9417 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9207 +num rollout transitions: 250000, reward mean: 8.9208 +num rollout transitions: 250000, reward mean: 8.9854 +num rollout transitions: 250000, reward mean: 8.9429 +num rollout transitions: 250000, reward mean: 8.9579 +num rollout transitions: 250000, reward mean: 8.9742 +----------------------------------------------------------------------------------- +| alpha | 0.352 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 3.14 | +| loss/actor | -992 | +| loss/alpha | -0.000785 | +| loss/critic | 78.1 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.74 | +| rollout_info/dynamics_penalty_mean | 0.000277 | +| rollout_info/dynamics_penalty_std | 0.016 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2050000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9658 +num rollout transitions: 250000, reward mean: 8.9698 +num rollout transitions: 250000, reward mean: 8.9917 +num rollout transitions: 250000, reward mean: 8.9259 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9021 +num rollout transitions: 250000, reward mean: 8.9409 +num rollout transitions: 250000, reward mean: 8.9934 +num rollout transitions: 250000, reward mean: 8.9378 +num rollout transitions: 250000, reward mean: 8.9569 +num rollout transitions: 250000, reward mean: 8.9479 +num rollout transitions: 250000, reward mean: 8.9162 +num rollout transitions: 250000, reward mean: 8.9292 +num rollout transitions: 250000, reward mean: 8.9272 +num rollout transitions: 250000, reward mean: 8.9591 +num rollout transitions: 250000, reward mean: 8.9657 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9676 +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9551 +num rollout transitions: 250000, reward mean: 8.9921 +num rollout transitions: 250000, reward mean: 8.9432 +num rollout transitions: 250000, reward mean: 8.9066 +num rollout transitions: 250000, reward mean: 8.9099 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9320 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9556 +num rollout transitions: 250000, reward mean: 8.9903 +num rollout transitions: 250000, reward mean: 8.9387 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9725 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9708 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9537 +num rollout transitions: 250000, reward mean: 8.9544 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9217 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9693 +num rollout transitions: 250000, reward mean: 8.9751 +num rollout transitions: 250000, reward mean: 8.9501 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9570 +----------------------------------------------------------------------------------- +| alpha | 0.348 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 12.8 | +| loss/actor | -990 | +| loss/alpha | -0.000121 | +| loss/critic | 85.5 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.87 | +| rollout_info/dynamics_penalty_mean | 0.000283 | +| rollout_info/dynamics_penalty_std | 0.0161 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2100000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9787 +num rollout transitions: 250000, reward mean: 8.9610 +num rollout transitions: 250000, reward mean: 8.9368 +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.9968 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9556 +num rollout transitions: 250000, reward mean: 8.9883 +num rollout transitions: 250000, reward mean: 8.9737 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9808 +num rollout transitions: 250000, reward mean: 8.9655 +num rollout transitions: 250000, reward mean: 8.9365 +num rollout transitions: 250000, reward mean: 8.9656 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.9510 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 8.9669 +num rollout transitions: 250000, reward mean: 8.9647 +num rollout transitions: 250000, reward mean: 8.9528 +num rollout transitions: 250000, reward mean: 8.9678 +num rollout transitions: 250000, reward mean: 8.9487 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9685 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9486 +num rollout transitions: 250000, reward mean: 8.9600 +num rollout transitions: 250000, reward mean: 8.9674 +num rollout transitions: 250000, reward mean: 8.9387 +num rollout transitions: 250000, reward mean: 8.9632 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9660 +num rollout transitions: 250000, reward mean: 9.0014 +num rollout transitions: 250000, reward mean: 8.9620 +num rollout transitions: 250000, reward mean: 8.9768 +num rollout transitions: 250000, reward mean: 8.9624 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9643 +num rollout transitions: 250000, reward mean: 8.9608 +num rollout transitions: 250000, reward mean: 8.9690 +num rollout transitions: 250000, reward mean: 8.9616 +num rollout transitions: 250000, reward mean: 8.9245 +num rollout transitions: 250000, reward mean: 8.9564 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9427 +num rollout transitions: 250000, reward mean: 8.9586 +num rollout transitions: 250000, reward mean: 8.9397 +---------------------------------------------------------------------------------- +| alpha | 0.351 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 97.3 | +| eval/normalized_episode_reward_std | 3.37 | +| loss/actor | -997 | +| loss/alpha | 0.000652 | +| loss/critic | 78.2 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.76 | +| rollout_info/dynamics_penalty_mean | 0.000258 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9510 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9562 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9852 +num rollout transitions: 250000, reward mean: 8.9679 +num rollout transitions: 250000, reward mean: 8.9558 +num rollout transitions: 250000, reward mean: 8.9779 +num rollout transitions: 250000, reward mean: 8.9676 +num rollout transitions: 250000, reward mean: 8.9737 +num rollout transitions: 250000, reward mean: 8.9288 +num rollout transitions: 250000, reward mean: 8.9665 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9676 +num rollout transitions: 250000, reward mean: 8.9972 +num rollout transitions: 250000, reward mean: 8.9875 +num rollout transitions: 250000, reward mean: 8.9496 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9676 +num rollout transitions: 250000, reward mean: 8.9329 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9209 +num rollout transitions: 250000, reward mean: 8.9560 +num rollout transitions: 250000, reward mean: 8.9573 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9498 +num rollout transitions: 250000, reward mean: 8.9787 +num rollout transitions: 250000, reward mean: 8.9608 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9819 +num rollout transitions: 250000, reward mean: 8.9659 +num rollout transitions: 250000, reward mean: 8.9496 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9723 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9681 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9416 +num rollout transitions: 250000, reward mean: 8.9303 +num rollout transitions: 250000, reward mean: 8.9467 +num rollout transitions: 250000, reward mean: 8.9659 +num rollout transitions: 250000, reward mean: 8.9726 +---------------------------------------------------------------------------------- +| alpha | 0.357 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 3.39 | +| loss/actor | -1e+03 | +| loss/alpha | 0.000325 | +| loss/critic | 75.6 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.69 | +| rollout_info/dynamics_penalty_mean | 0.000277 | +| rollout_info/dynamics_penalty_std | 0.016 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9552 +num rollout transitions: 250000, reward mean: 8.9818 +num rollout transitions: 250000, reward mean: 8.9576 +num rollout transitions: 250000, reward mean: 8.9634 +num rollout transitions: 250000, reward mean: 8.9628 +num rollout transitions: 250000, reward mean: 8.9637 +num rollout transitions: 250000, reward mean: 8.9008 +num rollout transitions: 250000, reward mean: 8.9366 +num rollout transitions: 250000, reward mean: 8.9572 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9784 +num rollout transitions: 250000, reward mean: 8.9268 +num rollout transitions: 250000, reward mean: 8.9627 +num rollout transitions: 250000, reward mean: 8.9484 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9413 +num rollout transitions: 250000, reward mean: 8.9610 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9858 +num rollout transitions: 250000, reward mean: 8.9751 +num rollout transitions: 250000, reward mean: 8.9274 +num rollout transitions: 250000, reward mean: 8.9646 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9785 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9632 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9502 +num rollout transitions: 250000, reward mean: 8.9576 +num rollout transitions: 250000, reward mean: 8.9581 +num rollout transitions: 250000, reward mean: 8.9505 +num rollout transitions: 250000, reward mean: 8.9803 +num rollout transitions: 250000, reward mean: 8.9216 +num rollout transitions: 250000, reward mean: 8.9652 +num rollout transitions: 250000, reward mean: 8.9307 +num rollout transitions: 250000, reward mean: 8.9453 +num rollout transitions: 250000, reward mean: 8.9628 +num rollout transitions: 250000, reward mean: 8.9750 +num rollout transitions: 250000, reward mean: 8.9635 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9888 +num rollout transitions: 250000, reward mean: 8.9658 +num rollout transitions: 250000, reward mean: 8.9468 +num rollout transitions: 250000, reward mean: 8.9856 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9209 +----------------------------------------------------------------------------------- +| alpha | 0.351 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.3 | +| eval/normalized_episode_reward_std | 3.49 | +| loss/actor | -999 | +| loss/alpha | -0.000175 | +| loss/critic | 78.5 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.74 | +| rollout_info/dynamics_penalty_mean | 0.00028 | +| rollout_info/dynamics_penalty_std | 0.0161 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2250000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9555 +num rollout transitions: 250000, reward mean: 8.9696 +num rollout transitions: 250000, reward mean: 8.9583 +num rollout transitions: 250000, reward mean: 8.9659 +num rollout transitions: 250000, reward mean: 8.9715 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9374 +num rollout transitions: 250000, reward mean: 8.9431 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.9901 +num rollout transitions: 250000, reward mean: 8.9635 +num rollout transitions: 250000, reward mean: 8.9838 +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9780 +num rollout transitions: 250000, reward mean: 8.9618 +num rollout transitions: 250000, reward mean: 8.9892 +num rollout transitions: 250000, reward mean: 8.9416 +num rollout transitions: 250000, reward mean: 8.9427 +num rollout transitions: 250000, reward mean: 8.9557 +num rollout transitions: 250000, reward mean: 8.9786 +num rollout transitions: 250000, reward mean: 8.9619 +num rollout transitions: 250000, reward mean: 8.9790 +num rollout transitions: 250000, reward mean: 8.9697 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9641 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9563 +num rollout transitions: 250000, reward mean: 8.9419 +num rollout transitions: 250000, reward mean: 8.9728 +num rollout transitions: 250000, reward mean: 8.9595 +num rollout transitions: 250000, reward mean: 8.9702 +num rollout transitions: 250000, reward mean: 8.9781 +num rollout transitions: 250000, reward mean: 8.9507 +num rollout transitions: 250000, reward mean: 8.9469 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9393 +num rollout transitions: 250000, reward mean: 8.9651 +num rollout transitions: 250000, reward mean: 8.9775 +num rollout transitions: 250000, reward mean: 8.9576 +num rollout transitions: 250000, reward mean: 8.9597 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9315 +num rollout transitions: 250000, reward mean: 8.9398 +---------------------------------------------------------------------------------- +| alpha | 0.354 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 4.84 | +| loss/actor | -1e+03 | +| loss/alpha | 0.00091 | +| loss/critic | 80.5 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.79 | +| rollout_info/dynamics_penalty_mean | 0.000254 | +| rollout_info/dynamics_penalty_std | 0.0153 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9728 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.9371 +num rollout transitions: 250000, reward mean: 8.9479 +num rollout transitions: 250000, reward mean: 8.9821 +num rollout transitions: 250000, reward mean: 8.9512 +num rollout transitions: 250000, reward mean: 8.9537 +num rollout transitions: 250000, reward mean: 8.9493 +num rollout transitions: 250000, reward mean: 8.9464 +num rollout transitions: 250000, reward mean: 8.9611 +num rollout transitions: 250000, reward mean: 8.9468 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9241 +num rollout transitions: 250000, reward mean: 8.9541 +num rollout transitions: 250000, reward mean: 8.9583 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9677 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9475 +num rollout transitions: 250000, reward mean: 8.9670 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9720 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9552 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9685 +num rollout transitions: 250000, reward mean: 8.9118 +num rollout transitions: 250000, reward mean: 8.9710 +num rollout transitions: 250000, reward mean: 8.9580 +num rollout transitions: 250000, reward mean: 8.9557 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9746 +num rollout transitions: 250000, reward mean: 8.9466 +num rollout transitions: 250000, reward mean: 8.9455 +num rollout transitions: 250000, reward mean: 8.9223 +num rollout transitions: 250000, reward mean: 8.9725 +num rollout transitions: 250000, reward mean: 8.9675 +num rollout transitions: 250000, reward mean: 8.9551 +num rollout transitions: 250000, reward mean: 8.9627 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9548 +num rollout transitions: 250000, reward mean: 8.9583 +num rollout transitions: 250000, reward mean: 8.9657 +num rollout transitions: 250000, reward mean: 8.9591 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9476 +num rollout transitions: 250000, reward mean: 8.9492 +---------------------------------------------------------------------------------- +| alpha | 0.356 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 11.8 | +| loss/actor | -1e+03 | +| loss/alpha | -0.00019 | +| loss/critic | 77.3 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.71 | +| rollout_info/dynamics_penalty_mean | 0.000275 | +| rollout_info/dynamics_penalty_std | 0.0159 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9674 +num rollout transitions: 250000, reward mean: 8.9463 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9674 +num rollout transitions: 250000, reward mean: 8.9293 +num rollout transitions: 250000, reward mean: 8.9573 +num rollout transitions: 250000, reward mean: 8.9514 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9819 +num rollout transitions: 250000, reward mean: 8.9752 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 8.9800 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9595 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9764 +num rollout transitions: 250000, reward mean: 8.9312 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9532 +num rollout transitions: 250000, reward mean: 8.9389 +num rollout transitions: 250000, reward mean: 8.9562 +num rollout transitions: 250000, reward mean: 8.9572 +num rollout transitions: 250000, reward mean: 8.9855 +num rollout transitions: 250000, reward mean: 8.9917 +num rollout transitions: 250000, reward mean: 8.8995 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.9560 +num rollout transitions: 250000, reward mean: 8.9420 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9660 +num rollout transitions: 250000, reward mean: 8.9779 +num rollout transitions: 250000, reward mean: 8.9654 +num rollout transitions: 250000, reward mean: 8.9480 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9531 +num rollout transitions: 250000, reward mean: 8.9750 +num rollout transitions: 250000, reward mean: 8.9622 +num rollout transitions: 250000, reward mean: 8.9658 +num rollout transitions: 250000, reward mean: 8.9661 +num rollout transitions: 250000, reward mean: 8.9720 +num rollout transitions: 250000, reward mean: 8.9563 +num rollout transitions: 250000, reward mean: 8.9684 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9684 +num rollout transitions: 250000, reward mean: 8.9480 +num rollout transitions: 250000, reward mean: 8.9619 +----------------------------------------------------------------------------------- +| alpha | 0.356 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 3.54 | +| loss/actor | -1e+03 | +| loss/alpha | -0.000124 | +| loss/critic | 79.4 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.79 | +| rollout_info/dynamics_penalty_mean | 0.000262 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2400000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9754 +num rollout transitions: 250000, reward mean: 8.9818 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9464 +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9464 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9286 +num rollout transitions: 250000, reward mean: 8.9581 +num rollout transitions: 250000, reward mean: 8.9566 +num rollout transitions: 250000, reward mean: 8.9764 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9569 +num rollout transitions: 250000, reward mean: 8.9474 +num rollout transitions: 250000, reward mean: 8.9506 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9644 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9552 +num rollout transitions: 250000, reward mean: 8.9664 +num rollout transitions: 250000, reward mean: 8.9578 +num rollout transitions: 250000, reward mean: 8.9751 +num rollout transitions: 250000, reward mean: 8.9723 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9662 +num rollout transitions: 250000, reward mean: 8.9193 +num rollout transitions: 250000, reward mean: 8.9686 +num rollout transitions: 250000, reward mean: 8.9544 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9279 +num rollout transitions: 250000, reward mean: 8.9645 +num rollout transitions: 250000, reward mean: 8.9623 +num rollout transitions: 250000, reward mean: 8.9354 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9646 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9576 +num rollout transitions: 250000, reward mean: 8.9624 +num rollout transitions: 250000, reward mean: 8.9314 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 9.0085 +num rollout transitions: 250000, reward mean: 8.9539 +---------------------------------------------------------------------------------- +| alpha | 0.357 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 3.41 | +| loss/actor | -1e+03 | +| loss/alpha | 0.000959 | +| loss/critic | 78.6 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.76 | +| rollout_info/dynamics_penalty_mean | 0.000258 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9514 +num rollout transitions: 250000, reward mean: 8.9700 +num rollout transitions: 250000, reward mean: 8.9283 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9380 +num rollout transitions: 250000, reward mean: 8.9705 +num rollout transitions: 250000, reward mean: 8.9566 +num rollout transitions: 250000, reward mean: 8.9353 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.9508 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9623 +num rollout transitions: 250000, reward mean: 8.9548 +num rollout transitions: 250000, reward mean: 8.9316 +num rollout transitions: 250000, reward mean: 8.9679 +num rollout transitions: 250000, reward mean: 8.9835 +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9131 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9802 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9384 +num rollout transitions: 250000, reward mean: 8.9187 +num rollout transitions: 250000, reward mean: 8.9325 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9548 +num rollout transitions: 250000, reward mean: 8.9651 +num rollout transitions: 250000, reward mean: 8.9803 +num rollout transitions: 250000, reward mean: 8.9417 +num rollout transitions: 250000, reward mean: 8.9531 +num rollout transitions: 250000, reward mean: 8.9688 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9606 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9610 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9531 +num rollout transitions: 250000, reward mean: 8.9455 +num rollout transitions: 250000, reward mean: 8.9241 +num rollout transitions: 250000, reward mean: 8.9542 +num rollout transitions: 250000, reward mean: 8.9474 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9535 +num rollout transitions: 250000, reward mean: 8.9720 +num rollout transitions: 250000, reward mean: 8.9278 +---------------------------------------------------------------------------------- +| alpha | 0.351 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 3.17 | +| loss/actor | -1e+03 | +| loss/alpha | 9.05e-05 | +| loss/critic | 85.4 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.9 | +| rollout_info/dynamics_penalty_mean | 0.000264 | +| rollout_info/dynamics_penalty_std | 0.0157 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9639 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9296 +num rollout transitions: 250000, reward mean: 8.9449 +num rollout transitions: 250000, reward mean: 8.9576 +num rollout transitions: 250000, reward mean: 8.9509 +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.9774 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9636 +num rollout transitions: 250000, reward mean: 8.9704 +num rollout transitions: 250000, reward mean: 8.9723 +num rollout transitions: 250000, reward mean: 8.9748 +num rollout transitions: 250000, reward mean: 8.9662 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9795 +num rollout transitions: 250000, reward mean: 8.9634 +num rollout transitions: 250000, reward mean: 8.9754 +num rollout transitions: 250000, reward mean: 8.9752 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9596 +num rollout transitions: 250000, reward mean: 8.9719 +num rollout transitions: 250000, reward mean: 8.9658 +num rollout transitions: 250000, reward mean: 8.9710 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9459 +num rollout transitions: 250000, reward mean: 8.9572 +num rollout transitions: 250000, reward mean: 8.9898 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9291 +num rollout transitions: 250000, reward mean: 8.9756 +num rollout transitions: 250000, reward mean: 8.9743 +num rollout transitions: 250000, reward mean: 8.9579 +num rollout transitions: 250000, reward mean: 8.9543 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.9474 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9595 +num rollout transitions: 250000, reward mean: 8.9542 +num rollout transitions: 250000, reward mean: 8.9636 +num rollout transitions: 250000, reward mean: 8.9743 +num rollout transitions: 250000, reward mean: 8.9608 +num rollout transitions: 250000, reward mean: 9.0051 +num rollout transitions: 250000, reward mean: 8.9605 +num rollout transitions: 250000, reward mean: 8.9718 +---------------------------------------------------------------------------------- +| alpha | 0.361 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 7.5 | +| loss/actor | -1e+03 | +| loss/alpha | 0.000357 | +| loss/critic | 80.2 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.78 | +| rollout_info/dynamics_penalty_mean | 0.000273 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9587 +num rollout transitions: 250000, reward mean: 8.9719 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9746 +num rollout transitions: 250000, reward mean: 8.9518 +num rollout transitions: 250000, reward mean: 8.9666 +num rollout transitions: 250000, reward mean: 8.9480 +num rollout transitions: 250000, reward mean: 8.9625 +num rollout transitions: 250000, reward mean: 8.9298 +num rollout transitions: 250000, reward mean: 8.9703 +num rollout transitions: 250000, reward mean: 8.9576 +num rollout transitions: 250000, reward mean: 8.9649 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.9586 +num rollout transitions: 250000, reward mean: 8.9663 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9757 +num rollout transitions: 250000, reward mean: 8.9688 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9509 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9571 +num rollout transitions: 250000, reward mean: 8.9678 +num rollout transitions: 250000, reward mean: 8.9768 +num rollout transitions: 250000, reward mean: 8.9882 +num rollout transitions: 250000, reward mean: 8.9386 +num rollout transitions: 250000, reward mean: 8.9625 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9851 +num rollout transitions: 250000, reward mean: 8.9564 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.9642 +num rollout transitions: 250000, reward mean: 8.9533 +num rollout transitions: 250000, reward mean: 8.9496 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9504 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9421 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.9463 +num rollout transitions: 250000, reward mean: 8.9269 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9489 +num rollout transitions: 250000, reward mean: 8.9452 +num rollout transitions: 250000, reward mean: 8.9449 +----------------------------------------------------------------------------------- +| alpha | 0.361 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 3.68 | +| loss/actor | -1.01e+03 | +| loss/alpha | -0.00257 | +| loss/critic | 80.1 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.83 | +| rollout_info/dynamics_penalty_mean | 0.000278 | +| rollout_info/dynamics_penalty_std | 0.0159 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2600000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9627 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9501 +num rollout transitions: 250000, reward mean: 8.9635 +num rollout transitions: 250000, reward mean: 8.9777 +num rollout transitions: 250000, reward mean: 8.9483 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.9797 +num rollout transitions: 250000, reward mean: 8.9637 +num rollout transitions: 250000, reward mean: 8.9345 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9708 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9581 +num rollout transitions: 250000, reward mean: 8.9556 +num rollout transitions: 250000, reward mean: 8.9580 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9386 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9590 +num rollout transitions: 250000, reward mean: 8.9582 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9442 +num rollout transitions: 250000, reward mean: 8.9806 +num rollout transitions: 250000, reward mean: 8.9609 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9729 +num rollout transitions: 250000, reward mean: 8.9611 +num rollout transitions: 250000, reward mean: 8.9661 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9780 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9481 +num rollout transitions: 250000, reward mean: 8.9750 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9388 +num rollout transitions: 250000, reward mean: 8.9537 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9702 +num rollout transitions: 250000, reward mean: 8.9684 +num rollout transitions: 250000, reward mean: 8.9704 +num rollout transitions: 250000, reward mean: 8.9542 +num rollout transitions: 250000, reward mean: 8.9521 +----------------------------------------------------------------------------------- +| alpha | 0.359 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 3.28 | +| loss/actor | -1.01e+03 | +| loss/alpha | 0.00353 | +| loss/critic | 78.3 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.76 | +| rollout_info/dynamics_penalty_mean | 0.000284 | +| rollout_info/dynamics_penalty_std | 0.0162 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2650000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9621 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9594 +num rollout transitions: 250000, reward mean: 8.9477 +num rollout transitions: 250000, reward mean: 8.9823 +num rollout transitions: 250000, reward mean: 8.9617 +num rollout transitions: 250000, reward mean: 8.9766 +num rollout transitions: 250000, reward mean: 8.9520 +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9304 +num rollout transitions: 250000, reward mean: 8.9505 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9634 +num rollout transitions: 250000, reward mean: 8.9665 +num rollout transitions: 250000, reward mean: 8.9606 +num rollout transitions: 250000, reward mean: 8.9820 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9698 +num rollout transitions: 250000, reward mean: 8.9667 +num rollout transitions: 250000, reward mean: 8.9476 +num rollout transitions: 250000, reward mean: 8.9731 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9695 +num rollout transitions: 250000, reward mean: 8.9543 +num rollout transitions: 250000, reward mean: 8.9956 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9613 +num rollout transitions: 250000, reward mean: 8.9367 +num rollout transitions: 250000, reward mean: 8.9715 +num rollout transitions: 250000, reward mean: 8.9598 +num rollout transitions: 250000, reward mean: 8.9481 +num rollout transitions: 250000, reward mean: 8.9638 +num rollout transitions: 250000, reward mean: 8.9649 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9594 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9812 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9757 +num rollout transitions: 250000, reward mean: 8.9369 +num rollout transitions: 250000, reward mean: 8.9762 +num rollout transitions: 250000, reward mean: 8.9498 +num rollout transitions: 250000, reward mean: 8.9715 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9684 +----------------------------------------------------------------------------------- +| alpha | 0.367 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 3.1 | +| loss/actor | -1.01e+03 | +| loss/alpha | -0.00129 | +| loss/critic | 81.5 | +| penalty/lcb_mean | 1.69 | +| penalty/lcb_std | 2.84 | +| rollout_info/dynamics_penalty_mean | 0.000261 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2700000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9429 +num rollout transitions: 250000, reward mean: 8.9638 +num rollout transitions: 250000, reward mean: 8.9560 +num rollout transitions: 250000, reward mean: 8.9888 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9706 +num rollout transitions: 250000, reward mean: 8.9565 +num rollout transitions: 250000, reward mean: 8.9586 +num rollout transitions: 250000, reward mean: 8.9626 +num rollout transitions: 250000, reward mean: 8.9586 +num rollout transitions: 250000, reward mean: 8.9511 +num rollout transitions: 250000, reward mean: 8.9422 +num rollout transitions: 250000, reward mean: 8.9798 +num rollout transitions: 250000, reward mean: 8.9609 +num rollout transitions: 250000, reward mean: 8.9639 +num rollout transitions: 250000, reward mean: 8.9754 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9453 +num rollout transitions: 250000, reward mean: 8.9732 +num rollout transitions: 250000, reward mean: 8.9758 +num rollout transitions: 250000, reward mean: 8.9384 +num rollout transitions: 250000, reward mean: 8.9685 +num rollout transitions: 250000, reward mean: 8.9343 +num rollout transitions: 250000, reward mean: 8.9644 +num rollout transitions: 250000, reward mean: 8.9788 +num rollout transitions: 250000, reward mean: 8.9477 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9439 +num rollout transitions: 250000, reward mean: 8.9635 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.9679 +num rollout transitions: 250000, reward mean: 8.9834 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9828 +num rollout transitions: 250000, reward mean: 8.9711 +num rollout transitions: 250000, reward mean: 8.9789 +num rollout transitions: 250000, reward mean: 8.9432 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9507 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9468 +num rollout transitions: 250000, reward mean: 8.9712 +num rollout transitions: 250000, reward mean: 8.9456 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.9684 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9906 +----------------------------------------------------------------------------------- +| alpha | 0.367 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 3.15 | +| loss/actor | -1.01e+03 | +| loss/alpha | 0.00148 | +| loss/critic | 80.7 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.81 | +| rollout_info/dynamics_penalty_mean | 0.000268 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2750000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9333 +num rollout transitions: 250000, reward mean: 8.9681 +num rollout transitions: 250000, reward mean: 8.9631 +num rollout transitions: 250000, reward mean: 8.9549 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9719 +num rollout transitions: 250000, reward mean: 8.9521 +num rollout transitions: 250000, reward mean: 8.9660 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9911 +num rollout transitions: 250000, reward mean: 8.9573 +num rollout transitions: 250000, reward mean: 8.9579 +num rollout transitions: 250000, reward mean: 8.9652 +num rollout transitions: 250000, reward mean: 8.9648 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9897 +num rollout transitions: 250000, reward mean: 8.9549 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9726 +num rollout transitions: 250000, reward mean: 8.9388 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9714 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9333 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9558 +num rollout transitions: 250000, reward mean: 8.9726 +num rollout transitions: 250000, reward mean: 8.9486 +num rollout transitions: 250000, reward mean: 8.9442 +num rollout transitions: 250000, reward mean: 8.9782 +num rollout transitions: 250000, reward mean: 8.9640 +num rollout transitions: 250000, reward mean: 8.9657 +num rollout transitions: 250000, reward mean: 8.9300 +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.9388 +num rollout transitions: 250000, reward mean: 8.9496 +num rollout transitions: 250000, reward mean: 8.9600 +num rollout transitions: 250000, reward mean: 8.9583 +num rollout transitions: 250000, reward mean: 8.9489 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9738 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9745 +num rollout transitions: 250000, reward mean: 8.9757 +num rollout transitions: 250000, reward mean: 8.9613 +num rollout transitions: 250000, reward mean: 8.9590 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9590 +----------------------------------------------------------------------------------- +| alpha | 0.368 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 10.4 | +| loss/actor | -1.01e+03 | +| loss/alpha | -0.000967 | +| loss/critic | 80.9 | +| penalty/lcb_mean | 1.69 | +| penalty/lcb_std | 2.84 | +| rollout_info/dynamics_penalty_mean | 0.000276 | +| rollout_info/dynamics_penalty_std | 0.0159 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2800000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9422 +num rollout transitions: 250000, reward mean: 8.9723 +num rollout transitions: 250000, reward mean: 8.9630 +num rollout transitions: 250000, reward mean: 8.9694 +num rollout transitions: 250000, reward mean: 8.9703 +num rollout transitions: 250000, reward mean: 8.9393 +num rollout transitions: 250000, reward mean: 8.9638 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9476 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9623 +num rollout transitions: 250000, reward mean: 8.9660 +num rollout transitions: 250000, reward mean: 8.9735 +num rollout transitions: 250000, reward mean: 8.9622 +num rollout transitions: 250000, reward mean: 8.9542 +num rollout transitions: 250000, reward mean: 8.9673 +num rollout transitions: 250000, reward mean: 8.9889 +num rollout transitions: 250000, reward mean: 8.9704 +num rollout transitions: 250000, reward mean: 8.9616 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9646 +num rollout transitions: 250000, reward mean: 8.9717 +num rollout transitions: 250000, reward mean: 8.9630 +num rollout transitions: 250000, reward mean: 8.9592 +num rollout transitions: 250000, reward mean: 8.9379 +num rollout transitions: 250000, reward mean: 8.9689 +num rollout transitions: 250000, reward mean: 8.9609 +num rollout transitions: 250000, reward mean: 8.9632 +num rollout transitions: 250000, reward mean: 8.9558 +num rollout transitions: 250000, reward mean: 8.9623 +num rollout transitions: 250000, reward mean: 8.9588 +num rollout transitions: 250000, reward mean: 8.9680 +num rollout transitions: 250000, reward mean: 8.9693 +num rollout transitions: 250000, reward mean: 8.9627 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9667 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9762 +num rollout transitions: 250000, reward mean: 8.9422 +num rollout transitions: 250000, reward mean: 8.9939 +num rollout transitions: 250000, reward mean: 8.9799 +num rollout transitions: 250000, reward mean: 8.9431 +num rollout transitions: 250000, reward mean: 8.9631 +num rollout transitions: 250000, reward mean: 8.9535 +num rollout transitions: 250000, reward mean: 8.9467 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9555 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9716 +----------------------------------------------------------------------------------- +| alpha | 0.368 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 9.95 | +| loss/actor | -1.01e+03 | +| loss/alpha | 0.0014 | +| loss/critic | 82.2 | +| penalty/lcb_mean | 1.69 | +| penalty/lcb_std | 2.84 | +| rollout_info/dynamics_penalty_mean | 0.000259 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2850000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9501 +num rollout transitions: 250000, reward mean: 8.9856 +num rollout transitions: 250000, reward mean: 8.9676 +num rollout transitions: 250000, reward mean: 8.9710 +num rollout transitions: 250000, reward mean: 8.9735 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9583 +num rollout transitions: 250000, reward mean: 8.9704 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9645 +num rollout transitions: 250000, reward mean: 8.9613 +num rollout transitions: 250000, reward mean: 8.9600 +num rollout transitions: 250000, reward mean: 8.9564 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9969 +num rollout transitions: 250000, reward mean: 8.9205 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9708 +num rollout transitions: 250000, reward mean: 8.9623 +num rollout transitions: 250000, reward mean: 8.9685 +num rollout transitions: 250000, reward mean: 8.9722 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9875 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.9583 +num rollout transitions: 250000, reward mean: 8.9781 +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9760 +num rollout transitions: 250000, reward mean: 8.9610 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.9494 +num rollout transitions: 250000, reward mean: 8.9720 +num rollout transitions: 250000, reward mean: 8.9489 +num rollout transitions: 250000, reward mean: 8.9752 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9735 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9533 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9719 +num rollout transitions: 250000, reward mean: 8.9687 +num rollout transitions: 250000, reward mean: 8.9697 +num rollout transitions: 250000, reward mean: 8.9447 +----------------------------------------------------------------------------------- +| alpha | 0.368 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 14.3 | +| loss/actor | -1.01e+03 | +| loss/alpha | -0.00188 | +| loss/critic | 81 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.84 | +| rollout_info/dynamics_penalty_mean | 0.000254 | +| rollout_info/dynamics_penalty_std | 0.0153 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2900000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9636 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9760 +num rollout transitions: 250000, reward mean: 8.9922 +num rollout transitions: 250000, reward mean: 8.9718 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9833 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9544 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9628 +num rollout transitions: 250000, reward mean: 8.9270 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9619 +num rollout transitions: 250000, reward mean: 8.9527 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9736 +num rollout transitions: 250000, reward mean: 8.9817 +num rollout transitions: 250000, reward mean: 8.9522 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9569 +num rollout transitions: 250000, reward mean: 8.9681 +num rollout transitions: 250000, reward mean: 8.9586 +num rollout transitions: 250000, reward mean: 8.9565 +num rollout transitions: 250000, reward mean: 8.9572 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9647 +num rollout transitions: 250000, reward mean: 8.9630 +num rollout transitions: 250000, reward mean: 8.9489 +num rollout transitions: 250000, reward mean: 8.9720 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9605 +num rollout transitions: 250000, reward mean: 8.9580 +num rollout transitions: 250000, reward mean: 8.9291 +num rollout transitions: 250000, reward mean: 8.9715 +num rollout transitions: 250000, reward mean: 8.9520 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.9644 +num rollout transitions: 250000, reward mean: 8.9623 +num rollout transitions: 250000, reward mean: 8.9826 +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9685 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9712 +num rollout transitions: 250000, reward mean: 8.9717 +num rollout transitions: 250000, reward mean: 8.9857 +----------------------------------------------------------------------------------- +| alpha | 0.368 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 10.7 | +| loss/actor | -1.01e+03 | +| loss/alpha | 0.000553 | +| loss/critic | 82.8 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.86 | +| rollout_info/dynamics_penalty_mean | 0.000266 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 2950000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9646 +num rollout transitions: 250000, reward mean: 8.9743 +num rollout transitions: 250000, reward mean: 8.9774 +num rollout transitions: 250000, reward mean: 8.9520 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9646 +num rollout transitions: 250000, reward mean: 8.9759 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.9650 +num rollout transitions: 250000, reward mean: 8.9899 +num rollout transitions: 250000, reward mean: 8.9536 +num rollout transitions: 250000, reward mean: 8.9449 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9682 +num rollout transitions: 250000, reward mean: 8.9445 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.9394 +num rollout transitions: 250000, reward mean: 8.9711 +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9461 +num rollout transitions: 250000, reward mean: 8.9569 +num rollout transitions: 250000, reward mean: 8.9799 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 8.9746 +num rollout transitions: 250000, reward mean: 8.9586 +num rollout transitions: 250000, reward mean: 8.9694 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9420 +num rollout transitions: 250000, reward mean: 8.9697 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9400 +num rollout transitions: 250000, reward mean: 8.9771 +num rollout transitions: 250000, reward mean: 8.9793 +num rollout transitions: 250000, reward mean: 8.9620 +num rollout transitions: 250000, reward mean: 8.9823 +num rollout transitions: 250000, reward mean: 8.9769 +num rollout transitions: 250000, reward mean: 8.9687 +num rollout transitions: 250000, reward mean: 8.9498 +num rollout transitions: 250000, reward mean: 8.9509 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9557 +num rollout transitions: 250000, reward mean: 8.9568 +num rollout transitions: 250000, reward mean: 8.9738 +num rollout transitions: 250000, reward mean: 8.9566 +num rollout transitions: 250000, reward mean: 8.9642 +num rollout transitions: 250000, reward mean: 8.9751 +num rollout transitions: 250000, reward mean: 8.9565 +num rollout transitions: 250000, reward mean: 8.9921 +num rollout transitions: 250000, reward mean: 8.9713 +num rollout transitions: 250000, reward mean: 8.9587 +----------------------------------------------------------------------------------- +| alpha | 0.366 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 6.84 | +| loss/actor | -1.01e+03 | +| loss/alpha | 0.000476 | +| loss/critic | 82.4 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.87 | +| rollout_info/dynamics_penalty_mean | 0.000259 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.96 | +| timestep | 3000000 | +----------------------------------------------------------------------------------- +total time: 133511.16s diff --git a/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-122101/record/dynamics_training_progress.csv b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-122101/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-122101/record/hyper_param.json b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-122101/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..6761a5ef625dbe02f3db275488e8a2d2a5f4f50b --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-122101/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/halfcheetah_medium_expert_72.5_vae.yaml", "algo_name": "mobile_gormpo", "task": "halfcheetah-medium-expert-v2", "seed": 123, "device": "cuda:7", "dataset_path": "/public/d4rl/sparse_datasets/halfcheetah_medium_expert_sparse_72.5.pkl", "dynamics_path": "log/halfcheetah-medium-expert-v2/mobile&penalty_coef=1.0&rollout_length=5/seed_123×tamp_26-0117-063103/model", "reward_penalty_coef": 0.3, "classifier_model_name": "/public/gormpo/models/halfcheetah_medium_expert_sparse_3/vae", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 5, "penalty_coef": 1.0, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.5, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "vae", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "vae"} diff --git a/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-122101/record/policy_training_progress.csv b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-122101/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..1a1b78ded297c68e27808354e25e66d688addb37 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-122101/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.5092469802635933,1000.0,0.0,24.629347813300125,19.518849771893695,-391.53271965292635,-0.3510507357736476,262.8571791229075,2.5625963130329383,3.767872209977699,0.0002686260199705697,0.015737345752857363,250000.0,8.846709080414788,50000 +0.6527629373753148,1000.0,0.0,38.212233130102405,23.67470807924101,-564.6794865393148,0.0031885760586149174,263.80415342834755,2.966348229565544,3.820940241310573,0.0002613913020855561,0.015585292203644933,250000.0,8.898805669889386,100000 +0.6370071728134133,1000.0,0.0,48.82404440594748,27.2663357156609,-699.8560675768955,-0.000939639359815022,217.6140987976068,2.808166308174163,3.7058655985116986,0.0002673737812531555,0.015683450548321286,250000.0,8.913125478271056,150000 +0.5958395690405418,1000.0,0.0,59.95464701207446,30.430627403034332,-781.0621136291578,-0.003335226206947145,183.80737370109395,2.582253915317034,3.5792016115832475,0.0002753947517600656,0.015929066341371086,250000.0,8.91688371389822,200000 +0.5398187292504347,1000.0,0.0,95.4240952408114,7.54096703993701,-844.7337557055607,-0.0038306241574976594,149.68339627235355,2.3124439065337357,3.3240350944948247,0.00025982423034440736,0.01540499730944121,250000.0,8.919448626640257,250000 +0.49855134805857587,1000.0,0.0,93.42476947144715,14.78045170274388,-874.9344382751542,-0.0016028708299715001,132.25739724449184,2.1540992547607547,3.1892134085130803,0.0002620361300269512,0.01543564701648936,250000.0,8.918959752401292,300000 +0.47308774027585765,1000.0,0.0,94.84682001207038,11.531269386410777,-899.9889500769067,-0.0020756243516504634,114.45493129150479,2.021582301061118,2.997146744881902,0.0002822366605207138,0.01610384267849952,250000.0,8.92210857379583,350000 +0.45334743123769927,1000.0,0.0,96.13239191016321,10.966061704916699,-917.1682233593713,-0.0020140857713855623,108.66677431762595,1.955712994830607,2.937181026978533,0.00026847482339069253,0.01575591475749518,250000.0,8.9275302420865,400000 +0.43459253629506067,1000.0,0.0,88.51066885463645,11.419089294280962,-921.1836921374509,-0.001520754339992988,109.49712723197844,1.941959081735631,3.0021115649438093,0.00026129383879810563,0.015468635611080033,250000.0,8.925707797067021,450000 +0.42281401423275955,1000.0,0.0,93.72451007916095,15.705907104102067,-930.268281705314,-0.0006043388151377517,105.42440392856508,1.8966981602144215,2.997290112373878,0.00026799133132429426,0.01566225240359512,250000.0,8.931304856322333,500000 +0.41565268611968137,1000.0,0.0,95.82930041702262,10.959222227944526,-939.7892551916535,0.001014108774177745,97.72837634662686,1.8530126563811276,2.9102312556541188,0.00028059389917809525,0.0161335497338732,250000.0,8.938001854219626,550000 +0.403699525281787,1000.0,0.0,97.4591168792015,11.12827746675573,-941.4984004772929,-0.0028776265700533794,96.8184091168211,1.8335422146797196,2.8828238867652276,0.0002562217196242512,0.015393209471174209,250000.0,8.935018243661512,600000 +0.40038901812494226,1000.0,0.0,94.96110033673025,17.949469310511404,-948.1876625109848,0.0005773089028894963,94.53401350898889,1.80844944916006,2.8675296164143367,0.0002749870213215054,0.015992079084339934,250000.0,8.937257690536011,650000 +0.39274757110118674,1000.0,0.0,92.29562846019238,3.499274804450604,-951.2909814514217,-0.0005566104005184215,92.76841828456963,1.7957892867732022,2.836012406831969,0.00027108345687158406,0.015799955608827233,250000.0,8.937757369909876,700000 +0.389827766659858,1000.0,0.0,93.27142911371332,12.21971715691788,-949.7004533129882,0.0007864895363897137,96.2774382466112,1.8169583765768917,2.9379520659124148,0.0002592641217275336,0.01537951756176212,250000.0,8.934994838370647,750000 +0.3873793018186095,1000.0,0.0,99.57220447166313,3.1961806044724166,-953.4410011035136,-0.0035501475371606545,91.40547734802188,1.774387276306171,2.86929707873105,0.0002706885949664563,0.015832144690386303,250000.0,8.93855000469403,800000 +0.38471447335720155,1000.0,0.0,87.68188763306206,21.07405692242984,-956.6371949560553,0.0009701544612832397,89.08035498138554,1.7566554062581046,2.8114554783082246,0.0002874510824891253,0.01631203747703495,250000.0,8.936611195193453,850000 +0.3806352545058762,1000.0,0.0,100.10093573035357,3.4917134494695543,-957.5414286181625,-0.0007985308922454992,87.61352699310261,1.748477064661977,2.776786977701143,0.00025718608662057667,0.015365792494077702,250000.0,8.939201683804267,900000 +0.38154172630667543,1000.0,0.0,95.44964899313464,16.585732322217133,-960.2676170532234,-0.0007606147248297722,87.63117334728229,1.7454240017390164,2.815399002690307,0.000292982289457023,0.01644311654077595,250000.0,8.94052041837362,950000 +0.37445152921557834,1000.0,0.0,99.13131872730918,3.259099769861265,-961.8012955236858,-0.0009437922016344856,84.39130285297284,1.7290673203635312,2.7612041790270547,0.0002594168907158636,0.015432914113628777,250000.0,8.942594293878416,1000000 +0.37569733042239184,1000.0,0.0,89.75515789155683,3.440866601800554,-962.6348215039008,0.0019754633665643134,86.93422994354196,1.7393999909544062,2.813293354643587,0.0002754563929836823,0.015920476934240587,250000.0,8.943844685655767,1050000 +0.37159930024266263,1000.0,0.0,100.26720007643763,7.616258556624543,-967.0285999304235,-0.001073434407897326,87.44750515743455,1.7255004037714023,2.8110152709841536,0.00025055761501098967,0.015182610868569335,250000.0,8.940876409247926,1100000 +0.37158364373862507,1000.0,0.0,95.90657705736105,16.259805674845023,-968.0295109265234,0.000443393495194618,85.74483168529495,1.7147031356000912,2.797777240048633,0.00026244359679590903,0.015535668259442444,250000.0,8.945642888964745,1150000 +0.37609793911039713,1000.0,0.0,99.50466986142644,6.206720762047125,-969.8003414916981,0.00039614921707660115,85.0706030209349,1.721641372313505,2.7998586848974445,0.00025801628058267766,0.015366514258615958,250000.0,8.946568332606994,1200000 +0.3745809584665332,1000.0,0.0,89.45540289679663,3.179700745136054,-969.4635332311957,-0.0008234431469626806,88.27823232814912,1.7288348467397652,2.84143365431312,0.0002620714248055964,0.015568816468706245,250000.0,8.946876914143084,1250000 +0.3670649679613075,1000.0,0.0,87.32045293968301,20.697619128863938,-969.6485908142155,-0.0017295316736400358,87.21440611545637,1.7086631653618676,2.8325245889341413,0.00026587313051810145,0.015607949455669158,250000.0,8.945094752026062,1300000 +0.369560746749639,1000.0,0.0,84.53803284590713,3.3577851181560305,-969.3295425036619,0.00012569334514439006,92.55852521427212,1.7351980210065865,2.930473705533757,0.00028181787016399193,0.01611279842151374,250000.0,8.946349636450858,1350000 +0.36881691627919677,1000.0,0.0,99.36416446724839,3.2891403540972073,-974.4540226391625,0.0026096314236893777,81.2455416074734,1.6780156922840965,2.7424247091782092,0.0002624197513008489,0.015493307241601162,250000.0,8.943262706037116,1400000 +0.36690922595738945,1000.0,0.0,91.84137247985157,22.25548598957564,-975.0353395263721,-0.0013666014960593852,88.35899591781694,1.7044666791093503,2.883514201369283,0.0002664392301695608,0.01570364457371398,250000.0,8.942178906223317,1450000 +0.3658819957047664,1000.0,0.0,102.03265957888574,3.4581626999636814,-976.8407187890668,0.000147561497725555,84.48453481414612,1.6915072344636974,2.8080927619278366,0.00026341131443548954,0.0155511594168024,250000.0,8.94193608099886,1500000 +0.3608538858109684,1000.0,0.0,101.17235156029089,2.885420555712861,-979.651040089108,-0.0009937367946282073,79.689051803894,1.6555585090327194,2.721372674504514,0.00028917519389960923,0.01630001352237772,250000.0,8.947059865466045,1550000 +0.3601143175500636,1000.0,0.0,100.53001185826282,3.2530510291189243,-981.6483123669487,-0.0006989672433398476,79.98182650020657,1.660094534152742,2.7344276479053353,0.0002696294486910104,0.01575108752134365,250000.0,8.949296170586955,1600000 +0.35679382735013937,1000.0,0.0,99.05220565406005,14.342052903241331,-982.1431392212081,5.035311970861147e-05,78.84725124290506,1.6455271623921266,2.7209143816768995,0.0002733866111446915,0.015805901912365642,250000.0,8.9493658631614,1650000 +0.3553304526627029,1000.0,0.0,98.47448684737516,3.073399867328371,-985.2654833630397,-0.0005580293138511348,82.90607525318089,1.6627190642571525,2.8101791279613524,0.0002493634200066328,0.015181193199882053,250000.0,8.945646961550572,1700000 +0.35743159500598853,1000.0,0.0,101.1173681017108,7.936719320663686,-980.6246439074747,0.0030050891544297523,106.93114810176907,1.7311189587056561,3.2498392102813995,0.0002692146257058997,0.015747990124443743,250000.0,8.947747849517896,1750000 +0.35321415485859325,1000.0,0.0,96.44531761625143,14.686136373272124,-981.9106149658195,-0.00366703940249978,79.18765295147003,1.6310848901200174,2.742378327864349,0.0002532035413428769,0.01518490750936948,250000.0,8.943943468910955,1800000 +0.35277950372874994,1000.0,0.0,103.26165265105803,10.587368248046069,-986.9761394751013,0.0011702385163493403,78.81689803146351,1.6317746647739486,2.73014716366059,0.0002467069366466999,0.015026085733064597,250000.0,8.948493247594222,1850000 +0.3511863115727893,1000.0,0.0,102.5241771461319,5.986285594501516,-987.61567296021,-0.0008805555361695579,80.40339624912299,1.6401674297070579,2.7858417784034897,0.00025936937966179095,0.015416494840033657,250000.0,8.947724166911858,1900000 +0.34818156589090854,1000.0,0.0,99.10777156151774,3.3508954535262268,-988.0049739209026,0.0008517224527895561,80.88876440227409,1.6292560636127056,2.7860337449086305,0.00026161291166514153,0.01553642860382928,250000.0,8.955366155430072,1950000 +0.351891207703952,1000.0,0.0,101.23113561144581,3.1594000665726636,-988.57017766601,0.0005381212700716952,81.7297975505831,1.6348028984260545,2.812544731913822,0.00027940330020368077,0.016058496997466375,250000.0,8.94786487071171,2000000 +0.3515281836354747,1000.0,0.0,100.90270151509246,3.144958729541134,-991.9383725512745,-0.0007854327745549648,78.12406625793386,1.636618301517964,2.7406152844822014,0.0002771883332521467,0.01597388870234079,250000.0,8.955390349000522,2050000 +0.34776497107208115,1000.0,0.0,102.2336938983852,12.830838433594927,-989.9206464782693,-0.00012112556530160292,85.53602380027665,1.6334023479962463,2.865545720994469,0.0002829117865410447,0.016133562597951248,250000.0,8.949288647724538,2100000 +0.3511892946058476,1000.0,0.0,97.33220207151454,3.373922154018606,-996.9765675158644,0.0006521848041005186,78.18689194583824,1.6282980061102,2.7642470647930972,0.00025812384904434906,0.015406462970212545,250000.0,8.958068186436526,2150000 +0.3568998014259306,1000.0,0.0,105.53604830398832,3.3904178929629643,-1000.1504632958986,0.000324757813159394,75.60282242775088,1.6254840306234128,2.6914281588744453,0.0002773781937360391,0.016035136025691017,250000.0,8.956011956594525,2200000 +0.3512102697205556,1000.0,0.0,99.26754318617128,3.4859658580011033,-998.7439924145514,-0.00017457223176954971,78.52902694854667,1.629067773880957,2.7433292569339223,0.0002798138747313711,0.016077141587736925,250000.0,8.955655879844844,2250000 +0.3536135471886368,1000.0,0.0,103.90228913484927,4.843266343885739,-1000.6653563830563,0.0009100512131303503,80.47583624965667,1.6436530286121436,2.7941910900902855,0.0002537251138340683,0.01534723215946449,250000.0,8.95741919306696,2300000 +0.3559289218091972,1000.0,0.0,103.43149259123739,11.808496452450507,-1001.7355634570379,-0.00019033085340634231,77.30257212200014,1.6287658619463268,2.7102024199700065,0.0002745414250689639,0.015934972607203394,250000.0,8.95274141682103,2350000 +0.35610892860769816,1000.0,0.0,101.0793371090255,3.538612221308485,-1003.7001866674728,-0.00012448781253770902,79.43949013763466,1.6509206805813401,2.790842116451273,0.00026159099354773776,0.015463198526209046,250000.0,8.956137044965105,2400000 +0.35694666972935296,1000.0,0.0,104.01599225435095,3.4077651967107983,-1004.0826767846673,0.000958958414429824,78.6490224240102,1.6446657089483756,2.761545264197605,0.000258017961518597,0.015405246462452234,250000.0,8.954672113514873,2450000 +0.3514931975454103,1000.0,0.0,104.23572679991351,3.1684965658448654,-1000.1399868957661,9.049817304130634e-05,85.4437689973841,1.6548947375142729,2.895442944284675,0.00026397256277091793,0.015658569461644147,250000.0,8.95015714928451,2500000 +0.3608373056596512,1000.0,0.0,104.35837260589031,7.504831425947543,-1004.2020517590252,0.0003565892244875328,80.23254651145841,1.660796654505728,2.783051377922334,0.00027312200755454596,0.01582215901682368,250000.0,8.96135434355157,2550000 +0.3610399420595155,1000.0,0.0,104.86710293852238,3.6789431830576023,-1006.6299072106954,-0.0025693858255259844,80.13818157890246,1.6665354282927438,2.830419885171653,0.000277750745511297,0.015938909719114075,250000.0,8.954786585674448,2600000 +0.3588295842528288,1000.0,0.0,105.44904678323562,3.281825738091674,-1005.2663296606446,0.0035302057293243923,78.33680464572994,1.6587072489666932,2.760274499213727,0.00028426545141022644,0.016248798186550867,250000.0,8.955758920136908,2650000 +0.36694890304208216,1000.0,0.0,105.09124098720326,3.0960993059945383,-1008.0497101477004,-0.0012927120116166943,81.51231284919675,1.6872004854297404,2.840375518455517,0.0002607781951724458,0.015449214463560524,250000.0,8.95900898430139,2700000 +0.3665212912297264,1000.0,0.0,105.02582733450446,3.149644267524512,-1007.219318231202,0.0014810661780648169,80.66616152191136,1.6787915455126698,2.8092141388583203,0.0002677626689758897,0.015627171041987596,250000.0,8.958538560070002,2750000 +0.3683182497352334,1000.0,0.0,103.97707546050643,10.35768904247259,-1007.4769229162505,-0.0009667522654123813,80.87263077186668,1.6909121167826453,2.8419860125577947,0.00027612106655818414,0.01591201237975903,250000.0,8.956427569806149,2800000 +0.3677936782032261,1000.0,0.0,104.46155326472694,9.95476054259123,-1007.38281293579,0.0013983167842775664,82.16664796508857,1.6901263534605506,2.8366580925094915,0.00025922229648251067,0.01537113922552623,250000.0,8.96011528975518,2850000 +0.3675041493314437,1000.0,0.0,103.29568292295322,14.296181563248478,-1007.2012369531269,-0.001884142428729669,80.99583232795739,1.6996495147204214,2.83881476768015,0.00025442667235967237,0.015291825908629972,250000.0,8.960330453918132,2900000 +0.3679523017722341,1000.0,0.0,104.95891482361148,10.724498936415417,-1007.8364745434445,0.0005529253545775871,82.8292617155843,1.7040991817522044,2.863277511118616,0.0002658576593628525,0.015617019828886962,250000.0,8.95891668840298,2950000 +0.36587727361202205,1000.0,0.0,105.34000539716335,6.840693887546066,-1007.4316909814428,0.00047597424887119963,82.41604610893272,1.6986213404917672,2.874082561496453,0.00025901357166163615,0.015401370073983234,250000.0,8.96137631700046,3000000 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-001614/record/consoleout_backup.txt b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-001614/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..c5e4da6ac01dd18000d7cbec7502e133375d9d68 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-001614/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 250000, reward mean: 8.7156 +num rollout transitions: 250000, reward mean: 8.7747 +num rollout transitions: 250000, reward mean: 8.7984 +num rollout transitions: 250000, reward mean: 8.7713 +num rollout transitions: 250000, reward mean: 8.7813 +num rollout transitions: 250000, reward mean: 8.7901 +num rollout transitions: 250000, reward mean: 8.8317 +num rollout transitions: 250000, reward mean: 8.7623 +num rollout transitions: 250000, reward mean: 8.8040 +num rollout transitions: 250000, reward mean: 8.7931 +num rollout transitions: 250000, reward mean: 8.7817 +num rollout transitions: 250000, reward mean: 8.7079 +num rollout transitions: 250000, reward mean: 8.8157 +num rollout transitions: 250000, reward mean: 8.7852 +num rollout transitions: 250000, reward mean: 8.7906 +num rollout transitions: 250000, reward mean: 8.8451 +num rollout transitions: 250000, reward mean: 8.8348 +num rollout transitions: 250000, reward mean: 8.7249 +num rollout transitions: 250000, reward mean: 8.8216 +num rollout transitions: 250000, reward mean: 8.8717 +num rollout transitions: 250000, reward mean: 8.8188 +num rollout transitions: 250000, reward mean: 8.8568 +num rollout transitions: 250000, reward mean: 8.8219 +num rollout transitions: 250000, reward mean: 8.7653 +num rollout transitions: 250000, reward mean: 8.7435 +num rollout transitions: 250000, reward mean: 8.8214 +num rollout transitions: 250000, reward mean: 8.8141 +num rollout transitions: 250000, reward mean: 8.8918 +num rollout transitions: 250000, reward mean: 8.8664 +num rollout transitions: 250000, reward mean: 8.7933 +num rollout transitions: 250000, reward mean: 8.8610 +num rollout transitions: 250000, reward mean: 8.8213 +num rollout transitions: 250000, reward mean: 8.8536 +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.8577 +num rollout transitions: 250000, reward mean: 8.8795 +num rollout transitions: 250000, reward mean: 8.8712 +num rollout transitions: 250000, reward mean: 8.8691 +num rollout transitions: 250000, reward mean: 8.8608 +num rollout transitions: 250000, reward mean: 8.8631 +num rollout transitions: 250000, reward mean: 8.8674 +num rollout transitions: 250000, reward mean: 8.8629 +num rollout transitions: 250000, reward mean: 8.9159 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.8311 +num rollout transitions: 250000, reward mean: 8.8519 +num rollout transitions: 250000, reward mean: 8.8497 +num rollout transitions: 250000, reward mean: 8.8521 +num rollout transitions: 250000, reward mean: 8.9452 +num rollout transitions: 250000, reward mean: 8.8900 +---------------------------------------------------------------------------------- +| alpha | 0.512 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 11.1 | +| eval/normalized_episode_reward_std | 13.7 | +| loss/actor | -384 | +| loss/alpha | -0.339 | +| loss/critic | 280 | +| penalty/lcb_mean | 2.54 | +| penalty/lcb_std | 3.67 | +| rollout_info/dynamics_penalty_mean | 0.00027 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.83 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9259 +num rollout transitions: 250000, reward mean: 8.8682 +num rollout transitions: 250000, reward mean: 8.8620 +num rollout transitions: 250000, reward mean: 8.8486 +num rollout transitions: 250000, reward mean: 8.9093 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.8746 +num rollout transitions: 250000, reward mean: 8.9178 +num rollout transitions: 250000, reward mean: 8.8879 +num rollout transitions: 250000, reward mean: 8.8831 +num rollout transitions: 250000, reward mean: 8.8690 +num rollout transitions: 250000, reward mean: 8.8978 +num rollout transitions: 250000, reward mean: 8.8491 +num rollout transitions: 250000, reward mean: 8.9004 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9168 +num rollout transitions: 250000, reward mean: 8.8481 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.8712 +num rollout transitions: 250000, reward mean: 8.9002 +num rollout transitions: 250000, reward mean: 8.9035 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.8565 +num rollout transitions: 250000, reward mean: 8.9475 +num rollout transitions: 250000, reward mean: 8.9139 +num rollout transitions: 250000, reward mean: 8.8753 +num rollout transitions: 250000, reward mean: 8.8413 +num rollout transitions: 250000, reward mean: 8.8889 +num rollout transitions: 250000, reward mean: 8.9159 +num rollout transitions: 250000, reward mean: 8.8619 +num rollout transitions: 250000, reward mean: 8.8760 +num rollout transitions: 250000, reward mean: 8.9038 +num rollout transitions: 250000, reward mean: 8.9004 +num rollout transitions: 250000, reward mean: 8.9180 +num rollout transitions: 250000, reward mean: 8.8903 +num rollout transitions: 250000, reward mean: 8.8829 +num rollout transitions: 250000, reward mean: 8.8819 +num rollout transitions: 250000, reward mean: 8.8804 +num rollout transitions: 250000, reward mean: 8.9095 +num rollout transitions: 250000, reward mean: 8.8800 +num rollout transitions: 250000, reward mean: 8.9464 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.9313 +num rollout transitions: 250000, reward mean: 8.8083 +num rollout transitions: 250000, reward mean: 8.8521 +num rollout transitions: 250000, reward mean: 8.8707 +num rollout transitions: 250000, reward mean: 8.9066 +num rollout transitions: 250000, reward mean: 8.9118 +num rollout transitions: 250000, reward mean: 8.9158 +num rollout transitions: 250000, reward mean: 8.9789 +---------------------------------------------------------------------------------- +| alpha | 0.696 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 69.9 | +| eval/normalized_episode_reward_std | 11.8 | +| loss/actor | -546 | +| loss/alpha | 0.00416 | +| loss/critic | 299 | +| penalty/lcb_mean | 3.11 | +| penalty/lcb_std | 3.95 | +| rollout_info/dynamics_penalty_mean | 0.000278 | +| rollout_info/dynamics_penalty_std | 0.0159 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.9 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.8483 +num rollout transitions: 250000, reward mean: 8.8978 +num rollout transitions: 250000, reward mean: 8.9020 +num rollout transitions: 250000, reward mean: 8.9212 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.8464 +num rollout transitions: 250000, reward mean: 8.9194 +num rollout transitions: 250000, reward mean: 8.8637 +num rollout transitions: 250000, reward mean: 8.9173 +num rollout transitions: 250000, reward mean: 8.9116 +num rollout transitions: 250000, reward mean: 8.9378 +num rollout transitions: 250000, reward mean: 8.8603 +num rollout transitions: 250000, reward mean: 8.8701 +num rollout transitions: 250000, reward mean: 8.8607 +num rollout transitions: 250000, reward mean: 8.8956 +num rollout transitions: 250000, reward mean: 8.9170 +num rollout transitions: 250000, reward mean: 8.9168 +num rollout transitions: 250000, reward mean: 8.9223 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.8995 +num rollout transitions: 250000, reward mean: 8.8878 +num rollout transitions: 250000, reward mean: 8.9087 +num rollout transitions: 250000, reward mean: 8.8720 +num rollout transitions: 250000, reward mean: 8.8900 +num rollout transitions: 250000, reward mean: 8.9455 +num rollout transitions: 250000, reward mean: 8.9241 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.8707 +num rollout transitions: 250000, reward mean: 8.8694 +num rollout transitions: 250000, reward mean: 8.9058 +num rollout transitions: 250000, reward mean: 8.9152 +num rollout transitions: 250000, reward mean: 8.9194 +num rollout transitions: 250000, reward mean: 8.9386 +num rollout transitions: 250000, reward mean: 8.8993 +num rollout transitions: 250000, reward mean: 8.9089 +num rollout transitions: 250000, reward mean: 8.8978 +num rollout transitions: 250000, reward mean: 8.9209 +num rollout transitions: 250000, reward mean: 8.9039 +num rollout transitions: 250000, reward mean: 8.8973 +num rollout transitions: 250000, reward mean: 8.9102 +num rollout transitions: 250000, reward mean: 8.9426 +num rollout transitions: 250000, reward mean: 8.8749 +num rollout transitions: 250000, reward mean: 8.9019 +num rollout transitions: 250000, reward mean: 8.8790 +num rollout transitions: 250000, reward mean: 8.9129 +num rollout transitions: 250000, reward mean: 8.8953 +num rollout transitions: 250000, reward mean: 8.9143 +num rollout transitions: 250000, reward mean: 8.8868 +num rollout transitions: 250000, reward mean: 8.9038 +---------------------------------------------------------------------------------- +| alpha | 0.692 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 72.5 | +| eval/normalized_episode_reward_std | 26 | +| loss/actor | -699 | +| loss/alpha | -0.00287 | +| loss/critic | 230 | +| penalty/lcb_mean | 2.85 | +| penalty/lcb_std | 3.66 | +| rollout_info/dynamics_penalty_mean | 0.000275 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.9 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9410 +num rollout transitions: 250000, reward mean: 8.9211 +num rollout transitions: 250000, reward mean: 8.8747 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9096 +num rollout transitions: 250000, reward mean: 8.9309 +num rollout transitions: 250000, reward mean: 8.9069 +num rollout transitions: 250000, reward mean: 8.9069 +num rollout transitions: 250000, reward mean: 8.8901 +num rollout transitions: 250000, reward mean: 8.8934 +num rollout transitions: 250000, reward mean: 8.8864 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.9163 +num rollout transitions: 250000, reward mean: 8.9113 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9099 +num rollout transitions: 250000, reward mean: 8.9640 +num rollout transitions: 250000, reward mean: 8.8945 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.8533 +num rollout transitions: 250000, reward mean: 8.8789 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9506 +num rollout transitions: 250000, reward mean: 8.9108 +num rollout transitions: 250000, reward mean: 8.9012 +num rollout transitions: 250000, reward mean: 8.8616 +num rollout transitions: 250000, reward mean: 8.9074 +num rollout transitions: 250000, reward mean: 8.9661 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.8771 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.8921 +num rollout transitions: 250000, reward mean: 8.9205 +num rollout transitions: 250000, reward mean: 8.8843 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.8518 +num rollout transitions: 250000, reward mean: 8.9496 +num rollout transitions: 250000, reward mean: 8.9142 +num rollout transitions: 250000, reward mean: 8.9138 +num rollout transitions: 250000, reward mean: 8.9353 +num rollout transitions: 250000, reward mean: 8.8803 +num rollout transitions: 250000, reward mean: 8.9149 +num rollout transitions: 250000, reward mean: 8.9282 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9133 +num rollout transitions: 250000, reward mean: 8.9302 +num rollout transitions: 250000, reward mean: 8.9094 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.8839 +num rollout transitions: 250000, reward mean: 8.9097 +---------------------------------------------------------------------------------- +| alpha | 0.619 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 89.3 | +| eval/normalized_episode_reward_std | 10.3 | +| loss/actor | -799 | +| loss/alpha | -0.00265 | +| loss/critic | 188 | +| penalty/lcb_mean | 2.56 | +| penalty/lcb_std | 3.55 | +| rollout_info/dynamics_penalty_mean | 0.000269 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.91 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9125 +num rollout transitions: 250000, reward mean: 8.9138 +num rollout transitions: 250000, reward mean: 8.9766 +num rollout transitions: 250000, reward mean: 8.8398 +num rollout transitions: 250000, reward mean: 8.8807 +num rollout transitions: 250000, reward mean: 8.9552 +num rollout transitions: 250000, reward mean: 8.9012 +num rollout transitions: 250000, reward mean: 8.9349 +num rollout transitions: 250000, reward mean: 8.9221 +num rollout transitions: 250000, reward mean: 8.9046 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9056 +num rollout transitions: 250000, reward mean: 8.8526 +num rollout transitions: 250000, reward mean: 8.8431 +num rollout transitions: 250000, reward mean: 8.9238 +num rollout transitions: 250000, reward mean: 8.8889 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9054 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.8979 +num rollout transitions: 250000, reward mean: 8.9149 +num rollout transitions: 250000, reward mean: 8.9068 +num rollout transitions: 250000, reward mean: 8.8854 +num rollout transitions: 250000, reward mean: 8.9125 +num rollout transitions: 250000, reward mean: 8.9022 +num rollout transitions: 250000, reward mean: 8.8837 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.8560 +num rollout transitions: 250000, reward mean: 8.8942 +num rollout transitions: 250000, reward mean: 8.9419 +num rollout transitions: 250000, reward mean: 8.9181 +num rollout transitions: 250000, reward mean: 8.9204 +num rollout transitions: 250000, reward mean: 8.9379 +num rollout transitions: 250000, reward mean: 8.9316 +num rollout transitions: 250000, reward mean: 8.9283 +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.9555 +num rollout transitions: 250000, reward mean: 8.9070 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.8825 +num rollout transitions: 250000, reward mean: 8.9024 +num rollout transitions: 250000, reward mean: 8.9273 +num rollout transitions: 250000, reward mean: 8.9378 +num rollout transitions: 250000, reward mean: 8.9199 +num rollout transitions: 250000, reward mean: 8.9281 +num rollout transitions: 250000, reward mean: 8.9000 +num rollout transitions: 250000, reward mean: 8.8923 +num rollout transitions: 250000, reward mean: 8.9226 +---------------------------------------------------------------------------------- +| alpha | 0.564 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 47.8 | +| eval/normalized_episode_reward_std | 26.1 | +| loss/actor | -851 | +| loss/alpha | -0.00257 | +| loss/critic | 161 | +| penalty/lcb_mean | 2.34 | +| penalty/lcb_std | 3.41 | +| rollout_info/dynamics_penalty_mean | 0.000266 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.91 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9674 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.8933 +num rollout transitions: 250000, reward mean: 8.8970 +num rollout transitions: 250000, reward mean: 8.8835 +num rollout transitions: 250000, reward mean: 8.8885 +num rollout transitions: 250000, reward mean: 8.9000 +num rollout transitions: 250000, reward mean: 8.9270 +num rollout transitions: 250000, reward mean: 8.9403 +num rollout transitions: 250000, reward mean: 8.9112 +num rollout transitions: 250000, reward mean: 8.9278 +num rollout transitions: 250000, reward mean: 8.9117 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.9822 +num rollout transitions: 250000, reward mean: 8.9131 +num rollout transitions: 250000, reward mean: 8.8609 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9260 +num rollout transitions: 250000, reward mean: 8.9257 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9187 +num rollout transitions: 250000, reward mean: 8.9374 +num rollout transitions: 250000, reward mean: 8.9046 +num rollout transitions: 250000, reward mean: 8.9151 +num rollout transitions: 250000, reward mean: 8.8982 +num rollout transitions: 250000, reward mean: 8.9215 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.8992 +num rollout transitions: 250000, reward mean: 8.9030 +num rollout transitions: 250000, reward mean: 8.9747 +num rollout transitions: 250000, reward mean: 8.9059 +num rollout transitions: 250000, reward mean: 8.8907 +num rollout transitions: 250000, reward mean: 8.9144 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9453 +num rollout transitions: 250000, reward mean: 8.8955 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9441 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9295 +num rollout transitions: 250000, reward mean: 8.8832 +num rollout transitions: 250000, reward mean: 8.9187 +num rollout transitions: 250000, reward mean: 8.9414 +num rollout transitions: 250000, reward mean: 8.9125 +num rollout transitions: 250000, reward mean: 8.8991 +num rollout transitions: 250000, reward mean: 8.9048 +num rollout transitions: 250000, reward mean: 8.9784 +num rollout transitions: 250000, reward mean: 8.8983 +---------------------------------------------------------------------------------- +| alpha | 0.524 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 90.7 | +| eval/normalized_episode_reward_std | 3.28 | +| loss/actor | -883 | +| loss/alpha | -0.00159 | +| loss/critic | 152 | +| penalty/lcb_mean | 2.25 | +| penalty/lcb_std | 3.42 | +| rollout_info/dynamics_penalty_mean | 0.00026 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.92 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.8909 +num rollout transitions: 250000, reward mean: 8.8626 +num rollout transitions: 250000, reward mean: 8.9309 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9234 +num rollout transitions: 250000, reward mean: 8.9077 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9331 +num rollout transitions: 250000, reward mean: 8.9073 +num rollout transitions: 250000, reward mean: 8.9271 +num rollout transitions: 250000, reward mean: 8.9337 +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.8974 +num rollout transitions: 250000, reward mean: 8.9091 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9148 +num rollout transitions: 250000, reward mean: 8.9128 +num rollout transitions: 250000, reward mean: 8.9235 +num rollout transitions: 250000, reward mean: 8.9103 +num rollout transitions: 250000, reward mean: 8.8729 +num rollout transitions: 250000, reward mean: 8.9326 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9059 +num rollout transitions: 250000, reward mean: 8.9059 +num rollout transitions: 250000, reward mean: 8.8897 +num rollout transitions: 250000, reward mean: 8.9201 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 8.8859 +num rollout transitions: 250000, reward mean: 8.9366 +num rollout transitions: 250000, reward mean: 8.8698 +num rollout transitions: 250000, reward mean: 8.8885 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9225 +num rollout transitions: 250000, reward mean: 8.9133 +num rollout transitions: 250000, reward mean: 8.9031 +num rollout transitions: 250000, reward mean: 8.8873 +num rollout transitions: 250000, reward mean: 8.9517 +num rollout transitions: 250000, reward mean: 8.9080 +num rollout transitions: 250000, reward mean: 8.9099 +num rollout transitions: 250000, reward mean: 8.9107 +num rollout transitions: 250000, reward mean: 8.8773 +num rollout transitions: 250000, reward mean: 8.8870 +num rollout transitions: 250000, reward mean: 8.9586 +num rollout transitions: 250000, reward mean: 8.9307 +num rollout transitions: 250000, reward mean: 8.9631 +num rollout transitions: 250000, reward mean: 8.8890 +---------------------------------------------------------------------------------- +| alpha | 0.491 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 91.4 | +| eval/normalized_episode_reward_std | 10.6 | +| loss/actor | -902 | +| loss/alpha | -0.00312 | +| loss/critic | 131 | +| penalty/lcb_mean | 2.1 | +| penalty/lcb_std | 3.2 | +| rollout_info/dynamics_penalty_mean | 0.000277 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.92 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9410 +num rollout transitions: 250000, reward mean: 8.9188 +num rollout transitions: 250000, reward mean: 8.9562 +num rollout transitions: 250000, reward mean: 8.8552 +num rollout transitions: 250000, reward mean: 8.9242 +num rollout transitions: 250000, reward mean: 8.9100 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9055 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9345 +num rollout transitions: 250000, reward mean: 8.9214 +num rollout transitions: 250000, reward mean: 8.9326 +num rollout transitions: 250000, reward mean: 8.9303 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9355 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9170 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.8647 +num rollout transitions: 250000, reward mean: 8.9598 +num rollout transitions: 250000, reward mean: 8.9201 +num rollout transitions: 250000, reward mean: 8.9097 +num rollout transitions: 250000, reward mean: 8.9374 +num rollout transitions: 250000, reward mean: 8.9218 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9088 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9182 +num rollout transitions: 250000, reward mean: 8.9234 +num rollout transitions: 250000, reward mean: 8.9331 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.8855 +num rollout transitions: 250000, reward mean: 8.9640 +num rollout transitions: 250000, reward mean: 8.8501 +num rollout transitions: 250000, reward mean: 8.8927 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9252 +num rollout transitions: 250000, reward mean: 8.9027 +num rollout transitions: 250000, reward mean: 8.9117 +num rollout transitions: 250000, reward mean: 8.9569 +num rollout transitions: 250000, reward mean: 8.8963 +num rollout transitions: 250000, reward mean: 8.9441 +num rollout transitions: 250000, reward mean: 8.8414 +---------------------------------------------------------------------------------- +| alpha | 0.465 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 96.1 | +| eval/normalized_episode_reward_std | 3.27 | +| loss/actor | -918 | +| loss/alpha | -0.001 | +| loss/critic | 112 | +| penalty/lcb_mean | 1.96 | +| penalty/lcb_std | 2.98 | +| rollout_info/dynamics_penalty_mean | 0.000256 | +| rollout_info/dynamics_penalty_std | 0.0152 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.92 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.8999 +num rollout transitions: 250000, reward mean: 8.9680 +num rollout transitions: 250000, reward mean: 8.9580 +num rollout transitions: 250000, reward mean: 8.9576 +num rollout transitions: 250000, reward mean: 8.9315 +num rollout transitions: 250000, reward mean: 8.8898 +num rollout transitions: 250000, reward mean: 8.9775 +num rollout transitions: 250000, reward mean: 8.8927 +num rollout transitions: 250000, reward mean: 8.9094 +num rollout transitions: 250000, reward mean: 8.8964 +num rollout transitions: 250000, reward mean: 8.8732 +num rollout transitions: 250000, reward mean: 8.9577 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9174 +num rollout transitions: 250000, reward mean: 8.9255 +num rollout transitions: 250000, reward mean: 8.9257 +num rollout transitions: 250000, reward mean: 8.8936 +num rollout transitions: 250000, reward mean: 8.9114 +num rollout transitions: 250000, reward mean: 8.9222 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9146 +num rollout transitions: 250000, reward mean: 8.9184 +num rollout transitions: 250000, reward mean: 8.9736 +num rollout transitions: 250000, reward mean: 8.9286 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9099 +num rollout transitions: 250000, reward mean: 8.8989 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9301 +num rollout transitions: 250000, reward mean: 8.9445 +num rollout transitions: 250000, reward mean: 8.9030 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9166 +num rollout transitions: 250000, reward mean: 8.9614 +num rollout transitions: 250000, reward mean: 8.9119 +num rollout transitions: 250000, reward mean: 8.9697 +num rollout transitions: 250000, reward mean: 8.8906 +num rollout transitions: 250000, reward mean: 8.9216 +num rollout transitions: 250000, reward mean: 8.9282 +num rollout transitions: 250000, reward mean: 8.9345 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9463 +num rollout transitions: 250000, reward mean: 8.9310 +num rollout transitions: 250000, reward mean: 8.9476 +num rollout transitions: 250000, reward mean: 8.8933 +num rollout transitions: 250000, reward mean: 8.9212 +num rollout transitions: 250000, reward mean: 8.9239 +---------------------------------------------------------------------------------- +| alpha | 0.447 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 92.9 | +| eval/normalized_episode_reward_std | 16.8 | +| loss/actor | -934 | +| loss/alpha | -0.00282 | +| loss/critic | 103 | +| penalty/lcb_mean | 1.89 | +| penalty/lcb_std | 2.84 | +| rollout_info/dynamics_penalty_mean | 0.000263 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9001 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9231 +num rollout transitions: 250000, reward mean: 8.9205 +num rollout transitions: 250000, reward mean: 8.9556 +num rollout transitions: 250000, reward mean: 8.9114 +num rollout transitions: 250000, reward mean: 8.8938 +num rollout transitions: 250000, reward mean: 8.9234 +num rollout transitions: 250000, reward mean: 8.9235 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.9071 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9248 +num rollout transitions: 250000, reward mean: 8.8918 +num rollout transitions: 250000, reward mean: 8.9158 +num rollout transitions: 250000, reward mean: 8.9294 +num rollout transitions: 250000, reward mean: 8.9220 +num rollout transitions: 250000, reward mean: 8.9261 +num rollout transitions: 250000, reward mean: 8.8997 +num rollout transitions: 250000, reward mean: 8.9171 +num rollout transitions: 250000, reward mean: 8.9160 +num rollout transitions: 250000, reward mean: 8.9145 +num rollout transitions: 250000, reward mean: 8.9139 +num rollout transitions: 250000, reward mean: 8.9459 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9139 +num rollout transitions: 250000, reward mean: 8.9198 +num rollout transitions: 250000, reward mean: 8.9887 +num rollout transitions: 250000, reward mean: 8.9222 +num rollout transitions: 250000, reward mean: 8.9140 +num rollout transitions: 250000, reward mean: 8.9297 +num rollout transitions: 250000, reward mean: 8.9062 +num rollout transitions: 250000, reward mean: 8.9269 +num rollout transitions: 250000, reward mean: 8.9432 +num rollout transitions: 250000, reward mean: 8.9212 +num rollout transitions: 250000, reward mean: 8.9135 +num rollout transitions: 250000, reward mean: 8.8868 +num rollout transitions: 250000, reward mean: 8.8846 +num rollout transitions: 250000, reward mean: 8.9544 +num rollout transitions: 250000, reward mean: 8.9177 +num rollout transitions: 250000, reward mean: 8.9396 +num rollout transitions: 250000, reward mean: 8.9206 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9157 +num rollout transitions: 250000, reward mean: 8.9312 +num rollout transitions: 250000, reward mean: 8.9264 +num rollout transitions: 250000, reward mean: 8.9436 +---------------------------------------------------------------------------------- +| alpha | 0.429 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 95.3 | +| eval/normalized_episode_reward_std | 11.6 | +| loss/actor | -940 | +| loss/alpha | -0.00136 | +| loss/critic | 99.8 | +| penalty/lcb_mean | 1.85 | +| penalty/lcb_std | 2.85 | +| rollout_info/dynamics_penalty_mean | 0.000266 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.92 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9190 +num rollout transitions: 250000, reward mean: 8.9142 +num rollout transitions: 250000, reward mean: 8.8956 +num rollout transitions: 250000, reward mean: 8.9100 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9262 +num rollout transitions: 250000, reward mean: 8.9417 +num rollout transitions: 250000, reward mean: 8.9253 +num rollout transitions: 250000, reward mean: 8.9223 +num rollout transitions: 250000, reward mean: 8.9118 +num rollout transitions: 250000, reward mean: 8.9129 +num rollout transitions: 250000, reward mean: 8.9646 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9429 +num rollout transitions: 250000, reward mean: 8.9160 +num rollout transitions: 250000, reward mean: 8.9213 +num rollout transitions: 250000, reward mean: 8.9521 +num rollout transitions: 250000, reward mean: 8.9104 +num rollout transitions: 250000, reward mean: 8.9211 +num rollout transitions: 250000, reward mean: 8.8957 +num rollout transitions: 250000, reward mean: 8.9594 +num rollout transitions: 250000, reward mean: 8.9380 +num rollout transitions: 250000, reward mean: 8.9332 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9258 +num rollout transitions: 250000, reward mean: 8.9064 +num rollout transitions: 250000, reward mean: 8.9484 +num rollout transitions: 250000, reward mean: 8.9239 +num rollout transitions: 250000, reward mean: 8.9268 +num rollout transitions: 250000, reward mean: 8.9140 +num rollout transitions: 250000, reward mean: 8.9222 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9378 +num rollout transitions: 250000, reward mean: 8.9042 +num rollout transitions: 250000, reward mean: 8.9644 +num rollout transitions: 250000, reward mean: 8.9329 +num rollout transitions: 250000, reward mean: 8.9227 +num rollout transitions: 250000, reward mean: 8.9089 +num rollout transitions: 250000, reward mean: 8.9012 +num rollout transitions: 250000, reward mean: 8.9649 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9078 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9442 +num rollout transitions: 250000, reward mean: 8.9532 +num rollout transitions: 250000, reward mean: 8.9131 +num rollout transitions: 250000, reward mean: 8.9007 +num rollout transitions: 250000, reward mean: 8.9512 +num rollout transitions: 250000, reward mean: 8.9328 +----------------------------------------------------------------------------------- +| alpha | 0.418 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 91.9 | +| eval/normalized_episode_reward_std | 20.1 | +| loss/actor | -944 | +| loss/alpha | -9.46e-05 | +| loss/critic | 96 | +| penalty/lcb_mean | 1.82 | +| penalty/lcb_std | 2.82 | +| rollout_info/dynamics_penalty_mean | 0.000259 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 550000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9513 +num rollout transitions: 250000, reward mean: 8.9456 +num rollout transitions: 250000, reward mean: 8.9132 +num rollout transitions: 250000, reward mean: 8.9120 +num rollout transitions: 250000, reward mean: 8.9099 +num rollout transitions: 250000, reward mean: 8.9108 +num rollout transitions: 250000, reward mean: 8.9067 +num rollout transitions: 250000, reward mean: 8.8999 +num rollout transitions: 250000, reward mean: 8.9744 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9175 +num rollout transitions: 250000, reward mean: 8.9453 +num rollout transitions: 250000, reward mean: 8.8836 +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.8789 +num rollout transitions: 250000, reward mean: 8.9572 +num rollout transitions: 250000, reward mean: 8.9186 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9044 +num rollout transitions: 250000, reward mean: 8.8999 +num rollout transitions: 250000, reward mean: 8.9240 +num rollout transitions: 250000, reward mean: 8.9219 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9304 +num rollout transitions: 250000, reward mean: 8.9144 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.9349 +num rollout transitions: 250000, reward mean: 8.9228 +num rollout transitions: 250000, reward mean: 8.9188 +num rollout transitions: 250000, reward mean: 8.9324 +num rollout transitions: 250000, reward mean: 8.9441 +num rollout transitions: 250000, reward mean: 8.9740 +num rollout transitions: 250000, reward mean: 8.9209 +num rollout transitions: 250000, reward mean: 8.9295 +num rollout transitions: 250000, reward mean: 8.9145 +num rollout transitions: 250000, reward mean: 8.9479 +num rollout transitions: 250000, reward mean: 8.9085 +num rollout transitions: 250000, reward mean: 8.9543 +num rollout transitions: 250000, reward mean: 8.9585 +num rollout transitions: 250000, reward mean: 8.8968 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9587 +num rollout transitions: 250000, reward mean: 8.9190 +num rollout transitions: 250000, reward mean: 8.9286 +num rollout transitions: 250000, reward mean: 8.9184 +num rollout transitions: 250000, reward mean: 8.9456 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9167 +num rollout transitions: 250000, reward mean: 8.9127 +----------------------------------------------------------------------------------- +| alpha | 0.408 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 94.3 | +| eval/normalized_episode_reward_std | 3.07 | +| loss/actor | -947 | +| loss/alpha | -0.000875 | +| loss/critic | 98.4 | +| penalty/lcb_mean | 1.82 | +| penalty/lcb_std | 2.88 | +| rollout_info/dynamics_penalty_mean | 0.000282 | +| rollout_info/dynamics_penalty_std | 0.016 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 600000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.8901 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9445 +num rollout transitions: 250000, reward mean: 8.9093 +num rollout transitions: 250000, reward mean: 8.9379 +num rollout transitions: 250000, reward mean: 8.9115 +num rollout transitions: 250000, reward mean: 8.9146 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.9588 +num rollout transitions: 250000, reward mean: 8.9533 +num rollout transitions: 250000, reward mean: 8.9579 +num rollout transitions: 250000, reward mean: 8.9210 +num rollout transitions: 250000, reward mean: 8.9248 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9254 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.9585 +num rollout transitions: 250000, reward mean: 8.9503 +num rollout transitions: 250000, reward mean: 8.9579 +num rollout transitions: 250000, reward mean: 8.9388 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.8641 +num rollout transitions: 250000, reward mean: 8.9084 +num rollout transitions: 250000, reward mean: 8.9292 +num rollout transitions: 250000, reward mean: 8.9620 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.8851 +num rollout transitions: 250000, reward mean: 8.8974 +num rollout transitions: 250000, reward mean: 8.9601 +num rollout transitions: 250000, reward mean: 8.9316 +num rollout transitions: 250000, reward mean: 8.9248 +num rollout transitions: 250000, reward mean: 8.9223 +num rollout transitions: 250000, reward mean: 8.9038 +num rollout transitions: 250000, reward mean: 8.9496 +num rollout transitions: 250000, reward mean: 8.9456 +num rollout transitions: 250000, reward mean: 8.8974 +num rollout transitions: 250000, reward mean: 8.9283 +num rollout transitions: 250000, reward mean: 8.9092 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9459 +num rollout transitions: 250000, reward mean: 8.9353 +num rollout transitions: 250000, reward mean: 8.9323 +num rollout transitions: 250000, reward mean: 8.9436 +num rollout transitions: 250000, reward mean: 8.8939 +num rollout transitions: 250000, reward mean: 8.9328 +num rollout transitions: 250000, reward mean: 8.9243 +num rollout transitions: 250000, reward mean: 8.9477 +---------------------------------------------------------------------------------- +| alpha | 0.399 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 92.1 | +| eval/normalized_episode_reward_std | 3.42 | +| loss/actor | -955 | +| loss/alpha | -0.0018 | +| loss/critic | 91.4 | +| penalty/lcb_mean | 1.77 | +| penalty/lcb_std | 2.8 | +| rollout_info/dynamics_penalty_mean | 0.000261 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.8873 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9102 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9096 +num rollout transitions: 250000, reward mean: 8.9571 +num rollout transitions: 250000, reward mean: 8.9130 +num rollout transitions: 250000, reward mean: 8.9066 +num rollout transitions: 250000, reward mean: 8.9388 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.8888 +num rollout transitions: 250000, reward mean: 8.9542 +num rollout transitions: 250000, reward mean: 8.9164 +num rollout transitions: 250000, reward mean: 8.9259 +num rollout transitions: 250000, reward mean: 8.9264 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9044 +num rollout transitions: 250000, reward mean: 8.9005 +num rollout transitions: 250000, reward mean: 8.9350 +num rollout transitions: 250000, reward mean: 8.8982 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9381 +num rollout transitions: 250000, reward mean: 8.9338 +num rollout transitions: 250000, reward mean: 8.9193 +num rollout transitions: 250000, reward mean: 8.8863 +num rollout transitions: 250000, reward mean: 8.9373 +num rollout transitions: 250000, reward mean: 8.9052 +num rollout transitions: 250000, reward mean: 8.9481 +num rollout transitions: 250000, reward mean: 8.9512 +num rollout transitions: 250000, reward mean: 8.9475 +num rollout transitions: 250000, reward mean: 8.9196 +num rollout transitions: 250000, reward mean: 8.9082 +num rollout transitions: 250000, reward mean: 8.9130 +num rollout transitions: 250000, reward mean: 8.9266 +num rollout transitions: 250000, reward mean: 8.9282 +num rollout transitions: 250000, reward mean: 8.9658 +num rollout transitions: 250000, reward mean: 8.9106 +num rollout transitions: 250000, reward mean: 8.9141 +num rollout transitions: 250000, reward mean: 8.8884 +num rollout transitions: 250000, reward mean: 8.9215 +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.9324 +num rollout transitions: 250000, reward mean: 8.9324 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9512 +num rollout transitions: 250000, reward mean: 8.9183 +num rollout transitions: 250000, reward mean: 8.9108 +----------------------------------------------------------------------------------- +| alpha | 0.393 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 98.3 | +| eval/normalized_episode_reward_std | 3.31 | +| loss/actor | -957 | +| loss/alpha | -0.000433 | +| loss/critic | 89.2 | +| penalty/lcb_mean | 1.75 | +| penalty/lcb_std | 2.76 | +| rollout_info/dynamics_penalty_mean | 0.000258 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 700000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9064 +num rollout transitions: 250000, reward mean: 8.9314 +num rollout transitions: 250000, reward mean: 8.9110 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9749 +num rollout transitions: 250000, reward mean: 8.9114 +num rollout transitions: 250000, reward mean: 8.9261 +num rollout transitions: 250000, reward mean: 8.9347 +num rollout transitions: 250000, reward mean: 8.9780 +num rollout transitions: 250000, reward mean: 8.9380 +num rollout transitions: 250000, reward mean: 8.8888 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9623 +num rollout transitions: 250000, reward mean: 8.9388 +num rollout transitions: 250000, reward mean: 8.9192 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9178 +num rollout transitions: 250000, reward mean: 8.9140 +num rollout transitions: 250000, reward mean: 8.9392 +num rollout transitions: 250000, reward mean: 8.9047 +num rollout transitions: 250000, reward mean: 8.9287 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9456 +num rollout transitions: 250000, reward mean: 8.9196 +num rollout transitions: 250000, reward mean: 8.9155 +num rollout transitions: 250000, reward mean: 8.9233 +num rollout transitions: 250000, reward mean: 8.9428 +num rollout transitions: 250000, reward mean: 8.9508 +num rollout transitions: 250000, reward mean: 8.9400 +num rollout transitions: 250000, reward mean: 8.9658 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9150 +num rollout transitions: 250000, reward mean: 8.9527 +num rollout transitions: 250000, reward mean: 8.9347 +num rollout transitions: 250000, reward mean: 8.9506 +num rollout transitions: 250000, reward mean: 8.9296 +num rollout transitions: 250000, reward mean: 8.9193 +num rollout transitions: 250000, reward mean: 8.9169 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9607 +num rollout transitions: 250000, reward mean: 8.9209 +num rollout transitions: 250000, reward mean: 8.9376 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9378 +num rollout transitions: 250000, reward mean: 8.9112 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.9243 +----------------------------------------------------------------------------------- +| alpha | 0.39 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.9 | +| eval/normalized_episode_reward_std | 3.84 | +| loss/actor | -962 | +| loss/alpha | -0.000942 | +| loss/critic | 87.3 | +| penalty/lcb_mean | 1.73 | +| penalty/lcb_std | 2.77 | +| rollout_info/dynamics_penalty_mean | 0.000254 | +| rollout_info/dynamics_penalty_std | 0.0151 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 750000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9084 +num rollout transitions: 250000, reward mean: 8.9343 +num rollout transitions: 250000, reward mean: 8.9251 +num rollout transitions: 250000, reward mean: 8.9686 +num rollout transitions: 250000, reward mean: 8.9259 +num rollout transitions: 250000, reward mean: 8.8937 +num rollout transitions: 250000, reward mean: 8.9017 +num rollout transitions: 250000, reward mean: 8.9409 +num rollout transitions: 250000, reward mean: 8.9561 +num rollout transitions: 250000, reward mean: 8.9677 +num rollout transitions: 250000, reward mean: 8.9174 +num rollout transitions: 250000, reward mean: 8.9233 +num rollout transitions: 250000, reward mean: 8.9225 +num rollout transitions: 250000, reward mean: 8.9042 +num rollout transitions: 250000, reward mean: 8.9449 +num rollout transitions: 250000, reward mean: 8.9310 +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9310 +num rollout transitions: 250000, reward mean: 8.9106 +num rollout transitions: 250000, reward mean: 8.9183 +num rollout transitions: 250000, reward mean: 8.9344 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9606 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.9242 +num rollout transitions: 250000, reward mean: 8.9146 +num rollout transitions: 250000, reward mean: 8.9148 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9793 +num rollout transitions: 250000, reward mean: 8.9077 +num rollout transitions: 250000, reward mean: 8.9649 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9068 +num rollout transitions: 250000, reward mean: 8.8920 +num rollout transitions: 250000, reward mean: 8.9475 +num rollout transitions: 250000, reward mean: 8.9551 +num rollout transitions: 250000, reward mean: 8.9675 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9329 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9106 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9344 +num rollout transitions: 250000, reward mean: 8.9148 +num rollout transitions: 250000, reward mean: 8.9365 +num rollout transitions: 250000, reward mean: 8.9150 +---------------------------------------------------------------------------------- +| alpha | 0.388 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.3 | +| eval/normalized_episode_reward_std | 4.82 | +| loss/actor | -965 | +| loss/alpha | 0.000738 | +| loss/critic | 93.9 | +| penalty/lcb_mean | 1.76 | +| penalty/lcb_std | 2.89 | +| rollout_info/dynamics_penalty_mean | 0.000262 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9271 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9432 +num rollout transitions: 250000, reward mean: 8.8963 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9325 +num rollout transitions: 250000, reward mean: 8.9004 +num rollout transitions: 250000, reward mean: 8.9477 +num rollout transitions: 250000, reward mean: 8.9229 +num rollout transitions: 250000, reward mean: 8.9410 +num rollout transitions: 250000, reward mean: 8.8979 +num rollout transitions: 250000, reward mean: 8.9184 +num rollout transitions: 250000, reward mean: 8.9585 +num rollout transitions: 250000, reward mean: 8.9170 +num rollout transitions: 250000, reward mean: 8.9745 +num rollout transitions: 250000, reward mean: 8.9171 +num rollout transitions: 250000, reward mean: 8.9329 +num rollout transitions: 250000, reward mean: 8.9261 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9449 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9165 +num rollout transitions: 250000, reward mean: 8.9416 +num rollout transitions: 250000, reward mean: 8.9202 +num rollout transitions: 250000, reward mean: 8.9469 +num rollout transitions: 250000, reward mean: 8.9541 +num rollout transitions: 250000, reward mean: 8.9608 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.9940 +num rollout transitions: 250000, reward mean: 8.9175 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9640 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.8929 +num rollout transitions: 250000, reward mean: 8.9608 +num rollout transitions: 250000, reward mean: 8.9487 +num rollout transitions: 250000, reward mean: 8.9797 +num rollout transitions: 250000, reward mean: 8.8944 +num rollout transitions: 250000, reward mean: 8.9125 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9024 +num rollout transitions: 250000, reward mean: 8.9135 +num rollout transitions: 250000, reward mean: 8.9558 +num rollout transitions: 250000, reward mean: 8.9751 +num rollout transitions: 250000, reward mean: 8.9393 +num rollout transitions: 250000, reward mean: 8.9489 +num rollout transitions: 250000, reward mean: 8.9431 +num rollout transitions: 250000, reward mean: 8.9018 +----------------------------------------------------------------------------------- +| alpha | 0.382 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 97.4 | +| eval/normalized_episode_reward_std | 3.07 | +| loss/actor | -968 | +| loss/alpha | -0.000555 | +| loss/critic | 85.8 | +| penalty/lcb_mean | 1.72 | +| penalty/lcb_std | 2.76 | +| rollout_info/dynamics_penalty_mean | 0.000257 | +| rollout_info/dynamics_penalty_std | 0.0153 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 850000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9180 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9449 +num rollout transitions: 250000, reward mean: 8.8893 +num rollout transitions: 250000, reward mean: 8.9541 +num rollout transitions: 250000, reward mean: 8.9190 +num rollout transitions: 250000, reward mean: 8.9139 +num rollout transitions: 250000, reward mean: 8.9191 +num rollout transitions: 250000, reward mean: 8.9135 +num rollout transitions: 250000, reward mean: 8.9449 +num rollout transitions: 250000, reward mean: 8.9249 +num rollout transitions: 250000, reward mean: 8.9394 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9192 +num rollout transitions: 250000, reward mean: 8.9609 +num rollout transitions: 250000, reward mean: 8.9493 +num rollout transitions: 250000, reward mean: 8.9201 +num rollout transitions: 250000, reward mean: 8.9245 +num rollout transitions: 250000, reward mean: 8.9067 +num rollout transitions: 250000, reward mean: 8.9777 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9796 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9381 +num rollout transitions: 250000, reward mean: 8.9075 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9091 +num rollout transitions: 250000, reward mean: 8.9637 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9093 +num rollout transitions: 250000, reward mean: 8.8999 +num rollout transitions: 250000, reward mean: 8.9339 +num rollout transitions: 250000, reward mean: 8.9114 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.9535 +num rollout transitions: 250000, reward mean: 8.9461 +num rollout transitions: 250000, reward mean: 8.9279 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9293 +num rollout transitions: 250000, reward mean: 8.9445 +num rollout transitions: 250000, reward mean: 8.9012 +num rollout transitions: 250000, reward mean: 8.9097 +num rollout transitions: 250000, reward mean: 8.9234 +num rollout transitions: 250000, reward mean: 8.9238 +num rollout transitions: 250000, reward mean: 8.9493 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9393 +num rollout transitions: 250000, reward mean: 8.9427 +num rollout transitions: 250000, reward mean: 8.9322 +---------------------------------------------------------------------------------- +| alpha | 0.376 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 6.94 | +| loss/actor | -970 | +| loss/alpha | 0.00133 | +| loss/critic | 84.6 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.74 | +| rollout_info/dynamics_penalty_mean | 0.000253 | +| rollout_info/dynamics_penalty_std | 0.0152 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9128 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9176 +num rollout transitions: 250000, reward mean: 8.9467 +num rollout transitions: 250000, reward mean: 8.9116 +num rollout transitions: 250000, reward mean: 8.8965 +num rollout transitions: 250000, reward mean: 8.9341 +num rollout transitions: 250000, reward mean: 8.9194 +num rollout transitions: 250000, reward mean: 8.9511 +num rollout transitions: 250000, reward mean: 8.9314 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9717 +num rollout transitions: 250000, reward mean: 8.9235 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.9541 +num rollout transitions: 250000, reward mean: 8.8992 +num rollout transitions: 250000, reward mean: 8.9703 +num rollout transitions: 250000, reward mean: 8.9204 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9302 +num rollout transitions: 250000, reward mean: 8.9804 +num rollout transitions: 250000, reward mean: 8.8913 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.9670 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9071 +num rollout transitions: 250000, reward mean: 8.9566 +num rollout transitions: 250000, reward mean: 8.9133 +num rollout transitions: 250000, reward mean: 8.9168 +num rollout transitions: 250000, reward mean: 8.9248 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9135 +num rollout transitions: 250000, reward mean: 8.9050 +num rollout transitions: 250000, reward mean: 8.9748 +num rollout transitions: 250000, reward mean: 8.9229 +num rollout transitions: 250000, reward mean: 8.9502 +num rollout transitions: 250000, reward mean: 8.9400 +num rollout transitions: 250000, reward mean: 8.9374 +num rollout transitions: 250000, reward mean: 8.8971 +num rollout transitions: 250000, reward mean: 8.9517 +num rollout transitions: 250000, reward mean: 8.9197 +num rollout transitions: 250000, reward mean: 8.9332 +num rollout transitions: 250000, reward mean: 8.8815 +num rollout transitions: 250000, reward mean: 8.9181 +num rollout transitions: 250000, reward mean: 8.9436 +num rollout transitions: 250000, reward mean: 8.9610 +---------------------------------------------------------------------------------- +| alpha | 0.371 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 97.8 | +| eval/normalized_episode_reward_std | 7.78 | +| loss/actor | -971 | +| loss/alpha | -0.00391 | +| loss/critic | 82.9 | +| penalty/lcb_mean | 1.69 | +| penalty/lcb_std | 2.71 | +| rollout_info/dynamics_penalty_mean | 0.000268 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.93 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9705 +num rollout transitions: 250000, reward mean: 8.9062 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9245 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9429 +num rollout transitions: 250000, reward mean: 8.9683 +num rollout transitions: 250000, reward mean: 8.9335 +num rollout transitions: 250000, reward mean: 8.9198 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9160 +num rollout transitions: 250000, reward mean: 8.9218 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9775 +num rollout transitions: 250000, reward mean: 8.9236 +num rollout transitions: 250000, reward mean: 8.9107 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9416 +num rollout transitions: 250000, reward mean: 8.9409 +num rollout transitions: 250000, reward mean: 8.9257 +num rollout transitions: 250000, reward mean: 8.9256 +num rollout transitions: 250000, reward mean: 8.9457 +num rollout transitions: 250000, reward mean: 8.9476 +num rollout transitions: 250000, reward mean: 8.9970 +num rollout transitions: 250000, reward mean: 8.9177 +num rollout transitions: 250000, reward mean: 8.9350 +num rollout transitions: 250000, reward mean: 8.9099 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9612 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9702 +num rollout transitions: 250000, reward mean: 8.9367 +num rollout transitions: 250000, reward mean: 8.9305 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9051 +num rollout transitions: 250000, reward mean: 8.9202 +num rollout transitions: 250000, reward mean: 8.9493 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9374 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9481 +num rollout transitions: 250000, reward mean: 8.9600 +num rollout transitions: 250000, reward mean: 8.8924 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9557 +num rollout transitions: 250000, reward mean: 8.9104 +num rollout transitions: 250000, reward mean: 8.8951 +num rollout transitions: 250000, reward mean: 8.9647 +----------------------------------------------------------------------------------- +| alpha | 0.374 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.5 | +| eval/normalized_episode_reward_std | 3.12 | +| loss/actor | -971 | +| loss/alpha | -9.71e-05 | +| loss/critic | 86.4 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.79 | +| rollout_info/dynamics_penalty_mean | 0.000254 | +| rollout_info/dynamics_penalty_std | 0.0152 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1000000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9720 +num rollout transitions: 250000, reward mean: 8.9114 +num rollout transitions: 250000, reward mean: 8.9008 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9624 +num rollout transitions: 250000, reward mean: 8.9181 +num rollout transitions: 250000, reward mean: 8.9176 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9206 +num rollout transitions: 250000, reward mean: 8.9392 +num rollout transitions: 250000, reward mean: 8.9165 +num rollout transitions: 250000, reward mean: 8.9613 +num rollout transitions: 250000, reward mean: 8.9343 +num rollout transitions: 250000, reward mean: 8.9266 +num rollout transitions: 250000, reward mean: 8.9657 +num rollout transitions: 250000, reward mean: 8.8960 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.9230 +num rollout transitions: 250000, reward mean: 8.9548 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9210 +num rollout transitions: 250000, reward mean: 8.9214 +num rollout transitions: 250000, reward mean: 8.9401 +num rollout transitions: 250000, reward mean: 8.8785 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.8999 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9304 +num rollout transitions: 250000, reward mean: 8.9662 +num rollout transitions: 250000, reward mean: 8.9560 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9274 +num rollout transitions: 250000, reward mean: 8.9329 +num rollout transitions: 250000, reward mean: 8.9223 +num rollout transitions: 250000, reward mean: 8.9610 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9355 +num rollout transitions: 250000, reward mean: 8.9506 +num rollout transitions: 250000, reward mean: 8.9198 +num rollout transitions: 250000, reward mean: 8.9387 +num rollout transitions: 250000, reward mean: 8.9452 +num rollout transitions: 250000, reward mean: 8.9092 +num rollout transitions: 250000, reward mean: 8.9703 +num rollout transitions: 250000, reward mean: 8.9569 +num rollout transitions: 250000, reward mean: 8.9239 +num rollout transitions: 250000, reward mean: 8.9233 +num rollout transitions: 250000, reward mean: 8.9200 +---------------------------------------------------------------------------------- +| alpha | 0.363 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 98.4 | +| eval/normalized_episode_reward_std | 13.3 | +| loss/actor | -971 | +| loss/alpha | 0.000388 | +| loss/critic | 80.6 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.71 | +| rollout_info/dynamics_penalty_mean | 0.000283 | +| rollout_info/dynamics_penalty_std | 0.0161 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9648 +num rollout transitions: 250000, reward mean: 8.9242 +num rollout transitions: 250000, reward mean: 8.9323 +num rollout transitions: 250000, reward mean: 8.9190 +num rollout transitions: 250000, reward mean: 8.9118 +num rollout transitions: 250000, reward mean: 8.9504 +num rollout transitions: 250000, reward mean: 8.9860 +num rollout transitions: 250000, reward mean: 8.9713 +num rollout transitions: 250000, reward mean: 8.9674 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9205 +num rollout transitions: 250000, reward mean: 8.9320 +num rollout transitions: 250000, reward mean: 8.9128 +num rollout transitions: 250000, reward mean: 8.9378 +num rollout transitions: 250000, reward mean: 8.9638 +num rollout transitions: 250000, reward mean: 8.9577 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9258 +num rollout transitions: 250000, reward mean: 8.9520 +num rollout transitions: 250000, reward mean: 8.8803 +num rollout transitions: 250000, reward mean: 8.9238 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9483 +num rollout transitions: 250000, reward mean: 8.9250 +num rollout transitions: 250000, reward mean: 8.9390 +num rollout transitions: 250000, reward mean: 8.9583 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9266 +num rollout transitions: 250000, reward mean: 8.9602 +num rollout transitions: 250000, reward mean: 8.9271 +num rollout transitions: 250000, reward mean: 8.9343 +num rollout transitions: 250000, reward mean: 8.9637 +num rollout transitions: 250000, reward mean: 8.9235 +num rollout transitions: 250000, reward mean: 8.9381 +num rollout transitions: 250000, reward mean: 8.9441 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9812 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.8976 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9439 +num rollout transitions: 250000, reward mean: 8.9208 +num rollout transitions: 250000, reward mean: 8.9279 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.9294 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9517 +num rollout transitions: 250000, reward mean: 8.9521 +---------------------------------------------------------------------------------- +| alpha | 0.359 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 97.1 | +| eval/normalized_episode_reward_std | 3.63 | +| loss/actor | -973 | +| loss/alpha | 0.000356 | +| loss/critic | 78.3 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.64 | +| rollout_info/dynamics_penalty_mean | 0.000262 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.9030 +num rollout transitions: 250000, reward mean: 8.9179 +num rollout transitions: 250000, reward mean: 8.9342 +num rollout transitions: 250000, reward mean: 8.9171 +num rollout transitions: 250000, reward mean: 8.9691 +num rollout transitions: 250000, reward mean: 8.9227 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9331 +num rollout transitions: 250000, reward mean: 8.9400 +num rollout transitions: 250000, reward mean: 8.9376 +num rollout transitions: 250000, reward mean: 8.9505 +num rollout transitions: 250000, reward mean: 8.9350 +num rollout transitions: 250000, reward mean: 8.9504 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9522 +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.9245 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9421 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9432 +num rollout transitions: 250000, reward mean: 8.9353 +num rollout transitions: 250000, reward mean: 8.9428 +num rollout transitions: 250000, reward mean: 8.9280 +num rollout transitions: 250000, reward mean: 8.9013 +num rollout transitions: 250000, reward mean: 8.9680 +num rollout transitions: 250000, reward mean: 8.9089 +num rollout transitions: 250000, reward mean: 8.9823 +num rollout transitions: 250000, reward mean: 8.9019 +num rollout transitions: 250000, reward mean: 8.9225 +num rollout transitions: 250000, reward mean: 8.9159 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9512 +num rollout transitions: 250000, reward mean: 8.9023 +num rollout transitions: 250000, reward mean: 8.9133 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.9708 +num rollout transitions: 250000, reward mean: 8.9239 +num rollout transitions: 250000, reward mean: 8.9520 +num rollout transitions: 250000, reward mean: 8.9551 +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.9413 +num rollout transitions: 250000, reward mean: 8.9536 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9071 +num rollout transitions: 250000, reward mean: 8.9361 +----------------------------------------------------------------------------------- +| alpha | 0.357 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 98 | +| eval/normalized_episode_reward_std | 9.89 | +| loss/actor | -976 | +| loss/alpha | -0.000845 | +| loss/critic | 78.4 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.66 | +| rollout_info/dynamics_penalty_mean | 0.00028 | +| rollout_info/dynamics_penalty_std | 0.016 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1150000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9389 +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9168 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9066 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.9285 +num rollout transitions: 250000, reward mean: 8.9233 +num rollout transitions: 250000, reward mean: 8.9353 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.9625 +num rollout transitions: 250000, reward mean: 8.9325 +num rollout transitions: 250000, reward mean: 8.9257 +num rollout transitions: 250000, reward mean: 8.9133 +num rollout transitions: 250000, reward mean: 8.9278 +num rollout transitions: 250000, reward mean: 8.9518 +num rollout transitions: 250000, reward mean: 8.9568 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.9217 +num rollout transitions: 250000, reward mean: 8.9167 +num rollout transitions: 250000, reward mean: 8.9161 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.8935 +num rollout transitions: 250000, reward mean: 8.9224 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9537 +num rollout transitions: 250000, reward mean: 8.9384 +num rollout transitions: 250000, reward mean: 8.9536 +num rollout transitions: 250000, reward mean: 8.9494 +num rollout transitions: 250000, reward mean: 8.9587 +num rollout transitions: 250000, reward mean: 8.9350 +num rollout transitions: 250000, reward mean: 8.9202 +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.9422 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9780 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9522 +num rollout transitions: 250000, reward mean: 8.9310 +num rollout transitions: 250000, reward mean: 8.9008 +num rollout transitions: 250000, reward mean: 8.9675 +num rollout transitions: 250000, reward mean: 8.9749 +num rollout transitions: 250000, reward mean: 8.8965 +---------------------------------------------------------------------------------- +| alpha | 0.354 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 96.5 | +| eval/normalized_episode_reward_std | 6.65 | +| loss/actor | -976 | +| loss/alpha | 0.00528 | +| loss/critic | 81.6 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.73 | +| rollout_info/dynamics_penalty_mean | 0.000289 | +| rollout_info/dynamics_penalty_std | 0.0162 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9326 +num rollout transitions: 250000, reward mean: 8.8923 +num rollout transitions: 250000, reward mean: 8.9578 +num rollout transitions: 250000, reward mean: 8.9166 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9331 +num rollout transitions: 250000, reward mean: 8.9513 +num rollout transitions: 250000, reward mean: 8.9217 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9368 +num rollout transitions: 250000, reward mean: 8.9298 +num rollout transitions: 250000, reward mean: 8.9339 +num rollout transitions: 250000, reward mean: 8.9494 +num rollout transitions: 250000, reward mean: 8.9323 +num rollout transitions: 250000, reward mean: 8.9341 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9558 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9244 +num rollout transitions: 250000, reward mean: 8.9195 +num rollout transitions: 250000, reward mean: 8.9350 +num rollout transitions: 250000, reward mean: 8.9318 +num rollout transitions: 250000, reward mean: 8.9325 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.8897 +num rollout transitions: 250000, reward mean: 8.9438 +num rollout transitions: 250000, reward mean: 8.9386 +num rollout transitions: 250000, reward mean: 8.8858 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.9110 +num rollout transitions: 250000, reward mean: 8.9854 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9611 +num rollout transitions: 250000, reward mean: 8.9654 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9257 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9380 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9721 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9236 +num rollout transitions: 250000, reward mean: 8.9279 +num rollout transitions: 250000, reward mean: 8.9548 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9536 +---------------------------------------------------------------------------------- +| alpha | 0.356 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.2 | +| eval/normalized_episode_reward_std | 3 | +| loss/actor | -975 | +| loss/alpha | -0.00442 | +| loss/critic | 88.8 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.86 | +| rollout_info/dynamics_penalty_mean | 0.000273 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9313 +num rollout transitions: 250000, reward mean: 8.9378 +num rollout transitions: 250000, reward mean: 8.9360 +num rollout transitions: 250000, reward mean: 8.9672 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9204 +num rollout transitions: 250000, reward mean: 8.9733 +num rollout transitions: 250000, reward mean: 8.9275 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9378 +num rollout transitions: 250000, reward mean: 8.9517 +num rollout transitions: 250000, reward mean: 8.9497 +num rollout transitions: 250000, reward mean: 8.9576 +num rollout transitions: 250000, reward mean: 8.9748 +num rollout transitions: 250000, reward mean: 8.9716 +num rollout transitions: 250000, reward mean: 8.9216 +num rollout transitions: 250000, reward mean: 8.9248 +num rollout transitions: 250000, reward mean: 8.9694 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9195 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9476 +num rollout transitions: 250000, reward mean: 8.9875 +num rollout transitions: 250000, reward mean: 8.9007 +num rollout transitions: 250000, reward mean: 8.9723 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9178 +num rollout transitions: 250000, reward mean: 8.9275 +num rollout transitions: 250000, reward mean: 8.9617 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9277 +num rollout transitions: 250000, reward mean: 8.9298 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.9287 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9902 +num rollout transitions: 250000, reward mean: 8.9709 +num rollout transitions: 250000, reward mean: 8.9165 +num rollout transitions: 250000, reward mean: 8.9248 +num rollout transitions: 250000, reward mean: 8.9132 +num rollout transitions: 250000, reward mean: 8.9476 +num rollout transitions: 250000, reward mean: 8.9486 +num rollout transitions: 250000, reward mean: 8.9601 +num rollout transitions: 250000, reward mean: 8.9655 +num rollout transitions: 250000, reward mean: 8.9333 +---------------------------------------------------------------------------------- +| alpha | 0.355 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 98.3 | +| eval/normalized_episode_reward_std | 13.2 | +| loss/actor | -978 | +| loss/alpha | 0.00073 | +| loss/critic | 78.7 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.71 | +| rollout_info/dynamics_penalty_mean | 0.000254 | +| rollout_info/dynamics_penalty_std | 0.0152 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9216 +num rollout transitions: 250000, reward mean: 8.9082 +num rollout transitions: 250000, reward mean: 8.9601 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9724 +num rollout transitions: 250000, reward mean: 8.9838 +num rollout transitions: 250000, reward mean: 8.9050 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9210 +num rollout transitions: 250000, reward mean: 8.9501 +num rollout transitions: 250000, reward mean: 8.9265 +num rollout transitions: 250000, reward mean: 8.9620 +num rollout transitions: 250000, reward mean: 8.9266 +num rollout transitions: 250000, reward mean: 8.9663 +num rollout transitions: 250000, reward mean: 8.9605 +num rollout transitions: 250000, reward mean: 8.9192 +num rollout transitions: 250000, reward mean: 8.9445 +num rollout transitions: 250000, reward mean: 8.9557 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9266 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9222 +num rollout transitions: 250000, reward mean: 8.9212 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9634 +num rollout transitions: 250000, reward mean: 8.9212 +num rollout transitions: 250000, reward mean: 8.9563 +num rollout transitions: 250000, reward mean: 8.9335 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9353 +num rollout transitions: 250000, reward mean: 8.9632 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9264 +num rollout transitions: 250000, reward mean: 8.9196 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.9585 +num rollout transitions: 250000, reward mean: 8.9388 +num rollout transitions: 250000, reward mean: 8.9461 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9339 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9251 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9666 +num rollout transitions: 250000, reward mean: 8.9266 +num rollout transitions: 250000, reward mean: 8.9231 +num rollout transitions: 250000, reward mean: 8.9542 +----------------------------------------------------------------------------------- +| alpha | 0.355 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 97.3 | +| eval/normalized_episode_reward_std | 15.8 | +| loss/actor | -982 | +| loss/alpha | -0.000953 | +| loss/critic | 76.9 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.64 | +| rollout_info/dynamics_penalty_mean | 0.00025 | +| rollout_info/dynamics_penalty_std | 0.015 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1350000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9472 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.9468 +num rollout transitions: 250000, reward mean: 8.9535 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9094 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9466 +num rollout transitions: 250000, reward mean: 8.9389 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9307 +num rollout transitions: 250000, reward mean: 8.9455 +num rollout transitions: 250000, reward mean: 8.9236 +num rollout transitions: 250000, reward mean: 8.9486 +num rollout transitions: 250000, reward mean: 8.9347 +num rollout transitions: 250000, reward mean: 8.9276 +num rollout transitions: 250000, reward mean: 8.9663 +num rollout transitions: 250000, reward mean: 8.9295 +num rollout transitions: 250000, reward mean: 8.9376 +num rollout transitions: 250000, reward mean: 8.9293 +num rollout transitions: 250000, reward mean: 8.9582 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9549 +num rollout transitions: 250000, reward mean: 8.8940 +num rollout transitions: 250000, reward mean: 8.9652 +num rollout transitions: 250000, reward mean: 8.9508 +num rollout transitions: 250000, reward mean: 8.9331 +num rollout transitions: 250000, reward mean: 8.9212 +num rollout transitions: 250000, reward mean: 8.9821 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9509 +num rollout transitions: 250000, reward mean: 8.9255 +num rollout transitions: 250000, reward mean: 8.9277 +num rollout transitions: 250000, reward mean: 8.9162 +num rollout transitions: 250000, reward mean: 8.9261 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9409 +num rollout transitions: 250000, reward mean: 8.9620 +num rollout transitions: 250000, reward mean: 8.9453 +num rollout transitions: 250000, reward mean: 8.9461 +num rollout transitions: 250000, reward mean: 8.9439 +num rollout transitions: 250000, reward mean: 8.9822 +num rollout transitions: 250000, reward mean: 8.9137 +num rollout transitions: 250000, reward mean: 8.9144 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9271 +num rollout transitions: 250000, reward mean: 8.9063 +num rollout transitions: 250000, reward mean: 8.9420 +num rollout transitions: 250000, reward mean: 8.9337 +---------------------------------------------------------------------------------- +| alpha | 0.35 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.1 | +| eval/normalized_episode_reward_std | 11.2 | +| loss/actor | -986 | +| loss/alpha | -0.00287 | +| loss/critic | 79.6 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.72 | +| rollout_info/dynamics_penalty_mean | 0.000274 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9518 +num rollout transitions: 250000, reward mean: 8.9538 +num rollout transitions: 250000, reward mean: 8.9388 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.9494 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9368 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9595 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9558 +num rollout transitions: 250000, reward mean: 8.9622 +num rollout transitions: 250000, reward mean: 8.9593 +num rollout transitions: 250000, reward mean: 8.9208 +num rollout transitions: 250000, reward mean: 8.8912 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9605 +num rollout transitions: 250000, reward mean: 8.9467 +num rollout transitions: 250000, reward mean: 8.9635 +num rollout transitions: 250000, reward mean: 8.9291 +num rollout transitions: 250000, reward mean: 8.9297 +num rollout transitions: 250000, reward mean: 8.9333 +num rollout transitions: 250000, reward mean: 8.9636 +num rollout transitions: 250000, reward mean: 8.9373 +num rollout transitions: 250000, reward mean: 8.9192 +num rollout transitions: 250000, reward mean: 8.9667 +num rollout transitions: 250000, reward mean: 8.9282 +num rollout transitions: 250000, reward mean: 8.9225 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9223 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9533 +num rollout transitions: 250000, reward mean: 8.9617 +num rollout transitions: 250000, reward mean: 8.9421 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 8.9381 +num rollout transitions: 250000, reward mean: 8.8971 +num rollout transitions: 250000, reward mean: 8.9479 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.9729 +num rollout transitions: 250000, reward mean: 8.9656 +num rollout transitions: 250000, reward mean: 8.9366 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9391 +---------------------------------------------------------------------------------- +| alpha | 0.344 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.6 | +| eval/normalized_episode_reward_std | 11.7 | +| loss/actor | -982 | +| loss/alpha | 0.00101 | +| loss/critic | 77.1 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.67 | +| rollout_info/dynamics_penalty_mean | 0.000273 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9282 +num rollout transitions: 250000, reward mean: 8.9374 +num rollout transitions: 250000, reward mean: 8.9709 +num rollout transitions: 250000, reward mean: 8.9787 +num rollout transitions: 250000, reward mean: 8.9174 +num rollout transitions: 250000, reward mean: 8.9102 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9276 +num rollout transitions: 250000, reward mean: 8.9452 +num rollout transitions: 250000, reward mean: 8.9513 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9461 +num rollout transitions: 250000, reward mean: 8.9512 +num rollout transitions: 250000, reward mean: 8.9691 +num rollout transitions: 250000, reward mean: 8.9787 +num rollout transitions: 250000, reward mean: 8.9243 +num rollout transitions: 250000, reward mean: 8.9084 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.9606 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9216 +num rollout transitions: 250000, reward mean: 8.9686 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9380 +num rollout transitions: 250000, reward mean: 8.9569 +num rollout transitions: 250000, reward mean: 8.9387 +num rollout transitions: 250000, reward mean: 8.9466 +num rollout transitions: 250000, reward mean: 8.9343 +num rollout transitions: 250000, reward mean: 8.9326 +num rollout transitions: 250000, reward mean: 8.9180 +num rollout transitions: 250000, reward mean: 8.9605 +num rollout transitions: 250000, reward mean: 8.9563 +num rollout transitions: 250000, reward mean: 8.9459 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9324 +num rollout transitions: 250000, reward mean: 8.9393 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9281 +num rollout transitions: 250000, reward mean: 8.9598 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9481 +num rollout transitions: 250000, reward mean: 8.9224 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.9498 +---------------------------------------------------------------------------------- +| alpha | 0.346 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 11.3 | +| loss/actor | -983 | +| loss/alpha | 0.00144 | +| loss/critic | 74.7 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.000262 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9678 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9220 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9503 +num rollout transitions: 250000, reward mean: 8.9745 +num rollout transitions: 250000, reward mean: 8.9410 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9113 +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.9263 +num rollout transitions: 250000, reward mean: 8.9735 +num rollout transitions: 250000, reward mean: 8.9277 +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.8853 +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9686 +num rollout transitions: 250000, reward mean: 8.9427 +num rollout transitions: 250000, reward mean: 8.9408 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9223 +num rollout transitions: 250000, reward mean: 8.9237 +num rollout transitions: 250000, reward mean: 8.9601 +num rollout transitions: 250000, reward mean: 8.9548 +num rollout transitions: 250000, reward mean: 8.9235 +num rollout transitions: 250000, reward mean: 8.9338 +num rollout transitions: 250000, reward mean: 8.9636 +num rollout transitions: 250000, reward mean: 8.9285 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9157 +num rollout transitions: 250000, reward mean: 8.9592 +num rollout transitions: 250000, reward mean: 8.9593 +num rollout transitions: 250000, reward mean: 8.9151 +num rollout transitions: 250000, reward mean: 8.9019 +num rollout transitions: 250000, reward mean: 8.9315 +num rollout transitions: 250000, reward mean: 8.9442 +num rollout transitions: 250000, reward mean: 8.9352 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9360 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9275 +num rollout transitions: 250000, reward mean: 8.9213 +num rollout transitions: 250000, reward mean: 8.9379 +num rollout transitions: 250000, reward mean: 8.9381 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9720 +---------------------------------------------------------------------------------- +| alpha | 0.348 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 97.6 | +| eval/normalized_episode_reward_std | 8.78 | +| loss/actor | -986 | +| loss/alpha | -0.00022 | +| loss/critic | 74.8 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.6 | +| rollout_info/dynamics_penalty_mean | 0.000249 | +| rollout_info/dynamics_penalty_std | 0.015 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9268 +num rollout transitions: 250000, reward mean: 8.8883 +num rollout transitions: 250000, reward mean: 8.9507 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9565 +num rollout transitions: 250000, reward mean: 8.9572 +num rollout transitions: 250000, reward mean: 8.9618 +num rollout transitions: 250000, reward mean: 8.9657 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9517 +num rollout transitions: 250000, reward mean: 8.9997 +num rollout transitions: 250000, reward mean: 8.9438 +num rollout transitions: 250000, reward mean: 8.9039 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9421 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9498 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9676 +num rollout transitions: 250000, reward mean: 8.9282 +num rollout transitions: 250000, reward mean: 8.9449 +num rollout transitions: 250000, reward mean: 8.9497 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9257 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9156 +num rollout transitions: 250000, reward mean: 8.9212 +num rollout transitions: 250000, reward mean: 8.9257 +num rollout transitions: 250000, reward mean: 8.9590 +num rollout transitions: 250000, reward mean: 8.9236 +num rollout transitions: 250000, reward mean: 8.9626 +num rollout transitions: 250000, reward mean: 8.9341 +num rollout transitions: 250000, reward mean: 8.9432 +num rollout transitions: 250000, reward mean: 8.9136 +num rollout transitions: 250000, reward mean: 8.9436 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9175 +num rollout transitions: 250000, reward mean: 8.9565 +num rollout transitions: 250000, reward mean: 8.8864 +num rollout transitions: 250000, reward mean: 8.9018 +num rollout transitions: 250000, reward mean: 8.9272 +num rollout transitions: 250000, reward mean: 8.8914 +num rollout transitions: 250000, reward mean: 8.9403 +num rollout transitions: 250000, reward mean: 8.9200 +num rollout transitions: 250000, reward mean: 8.9213 +num rollout transitions: 250000, reward mean: 8.9518 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9513 +num rollout transitions: 250000, reward mean: 8.9498 +---------------------------------------------------------------------------------- +| alpha | 0.346 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 4.74 | +| loss/actor | -985 | +| loss/alpha | 0.0001 | +| loss/critic | 80.5 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.73 | +| rollout_info/dynamics_penalty_mean | 0.000264 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9494 +num rollout transitions: 250000, reward mean: 8.9094 +num rollout transitions: 250000, reward mean: 8.9189 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9139 +num rollout transitions: 250000, reward mean: 8.9724 +num rollout transitions: 250000, reward mean: 8.9209 +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.9453 +num rollout transitions: 250000, reward mean: 8.9360 +num rollout transitions: 250000, reward mean: 8.9303 +num rollout transitions: 250000, reward mean: 8.9304 +num rollout transitions: 250000, reward mean: 8.9249 +num rollout transitions: 250000, reward mean: 8.9285 +num rollout transitions: 250000, reward mean: 8.9505 +num rollout transitions: 250000, reward mean: 8.9693 +num rollout transitions: 250000, reward mean: 8.9244 +num rollout transitions: 250000, reward mean: 8.9269 +num rollout transitions: 250000, reward mean: 8.9222 +num rollout transitions: 250000, reward mean: 8.9253 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9658 +num rollout transitions: 250000, reward mean: 8.9328 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9677 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9138 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9245 +num rollout transitions: 250000, reward mean: 8.9409 +num rollout transitions: 250000, reward mean: 8.9596 +num rollout transitions: 250000, reward mean: 8.9342 +num rollout transitions: 250000, reward mean: 8.9474 +num rollout transitions: 250000, reward mean: 8.9320 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9419 +num rollout transitions: 250000, reward mean: 8.9565 +num rollout transitions: 250000, reward mean: 8.9192 +num rollout transitions: 250000, reward mean: 8.9168 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9309 +num rollout transitions: 250000, reward mean: 8.9575 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9410 +num rollout transitions: 250000, reward mean: 8.9514 +num rollout transitions: 250000, reward mean: 8.9560 +num rollout transitions: 250000, reward mean: 8.9692 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9441 +---------------------------------------------------------------------------------- +| alpha | 0.342 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 7.89 | +| loss/actor | -983 | +| loss/alpha | -0.00151 | +| loss/critic | 86.6 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.84 | +| rollout_info/dynamics_penalty_mean | 0.000244 | +| rollout_info/dynamics_penalty_std | 0.0148 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.9532 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9018 +num rollout transitions: 250000, reward mean: 8.9353 +num rollout transitions: 250000, reward mean: 8.9438 +num rollout transitions: 250000, reward mean: 8.9532 +num rollout transitions: 250000, reward mean: 8.9513 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9286 +num rollout transitions: 250000, reward mean: 8.9568 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9276 +num rollout transitions: 250000, reward mean: 8.9461 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9160 +num rollout transitions: 250000, reward mean: 8.9394 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9426 +num rollout transitions: 250000, reward mean: 8.9549 +num rollout transitions: 250000, reward mean: 8.9339 +num rollout transitions: 250000, reward mean: 8.9528 +num rollout transitions: 250000, reward mean: 8.9079 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9277 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9419 +num rollout transitions: 250000, reward mean: 8.9816 +num rollout transitions: 250000, reward mean: 8.9401 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9294 +num rollout transitions: 250000, reward mean: 8.9256 +num rollout transitions: 250000, reward mean: 8.9538 +num rollout transitions: 250000, reward mean: 8.9577 +num rollout transitions: 250000, reward mean: 8.9503 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9221 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9324 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9654 +num rollout transitions: 250000, reward mean: 8.9214 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9329 +num rollout transitions: 250000, reward mean: 8.9464 +num rollout transitions: 250000, reward mean: 8.9546 +---------------------------------------------------------------------------------- +| alpha | 0.344 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 90.9 | +| eval/normalized_episode_reward_std | 3.1 | +| loss/actor | -987 | +| loss/alpha | 0.00215 | +| loss/critic | 76 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.66 | +| rollout_info/dynamics_penalty_mean | 0.000269 | +| rollout_info/dynamics_penalty_std | 0.0157 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9151 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.9498 +num rollout transitions: 250000, reward mean: 8.8962 +num rollout transitions: 250000, reward mean: 8.9422 +num rollout transitions: 250000, reward mean: 8.9957 +num rollout transitions: 250000, reward mean: 8.9212 +num rollout transitions: 250000, reward mean: 8.9353 +num rollout transitions: 250000, reward mean: 8.9663 +num rollout transitions: 250000, reward mean: 8.9622 +num rollout transitions: 250000, reward mean: 8.9403 +num rollout transitions: 250000, reward mean: 8.9527 +num rollout transitions: 250000, reward mean: 8.9541 +num rollout transitions: 250000, reward mean: 8.9514 +num rollout transitions: 250000, reward mean: 8.9501 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9511 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9439 +num rollout transitions: 250000, reward mean: 8.9670 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9339 +num rollout transitions: 250000, reward mean: 8.9719 +num rollout transitions: 250000, reward mean: 8.9708 +num rollout transitions: 250000, reward mean: 8.9126 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9459 +num rollout transitions: 250000, reward mean: 8.9670 +num rollout transitions: 250000, reward mean: 8.9139 +num rollout transitions: 250000, reward mean: 8.9833 +num rollout transitions: 250000, reward mean: 8.9063 +num rollout transitions: 250000, reward mean: 8.9239 +num rollout transitions: 250000, reward mean: 8.9679 +num rollout transitions: 250000, reward mean: 8.9501 +num rollout transitions: 250000, reward mean: 8.9573 +num rollout transitions: 250000, reward mean: 8.9404 +num rollout transitions: 250000, reward mean: 8.9315 +num rollout transitions: 250000, reward mean: 8.9439 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9439 +num rollout transitions: 250000, reward mean: 8.9453 +num rollout transitions: 250000, reward mean: 8.9267 +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9175 +num rollout transitions: 250000, reward mean: 8.9521 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9536 +---------------------------------------------------------------------------------- +| alpha | 0.347 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 13.9 | +| loss/actor | -988 | +| loss/alpha | -0.00128 | +| loss/critic | 76.5 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.68 | +| rollout_info/dynamics_penalty_mean | 0.000272 | +| rollout_info/dynamics_penalty_std | 0.0157 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9568 +num rollout transitions: 250000, reward mean: 8.9429 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9333 +num rollout transitions: 250000, reward mean: 8.9138 +num rollout transitions: 250000, reward mean: 8.9410 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9228 +num rollout transitions: 250000, reward mean: 8.9739 +num rollout transitions: 250000, reward mean: 8.9273 +num rollout transitions: 250000, reward mean: 8.9709 +num rollout transitions: 250000, reward mean: 8.9475 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9521 +num rollout transitions: 250000, reward mean: 8.9494 +num rollout transitions: 250000, reward mean: 8.9658 +num rollout transitions: 250000, reward mean: 8.9602 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9797 +num rollout transitions: 250000, reward mean: 8.9441 +num rollout transitions: 250000, reward mean: 8.9295 +num rollout transitions: 250000, reward mean: 8.9829 +num rollout transitions: 250000, reward mean: 8.9256 +num rollout transitions: 250000, reward mean: 8.9298 +num rollout transitions: 250000, reward mean: 8.9093 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9709 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9184 +num rollout transitions: 250000, reward mean: 8.9429 +num rollout transitions: 250000, reward mean: 8.9668 +num rollout transitions: 250000, reward mean: 8.9602 +num rollout transitions: 250000, reward mean: 8.9673 +num rollout transitions: 250000, reward mean: 8.9452 +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9336 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9153 +num rollout transitions: 250000, reward mean: 8.9677 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9576 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9324 +num rollout transitions: 250000, reward mean: 8.9493 +----------------------------------------------------------------------------------- +| alpha | 0.347 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 96.9 | +| eval/normalized_episode_reward_std | 3.26 | +| loss/actor | -993 | +| loss/alpha | -0.000222 | +| loss/critic | 75.1 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.66 | +| rollout_info/dynamics_penalty_mean | 0.000258 | +| rollout_info/dynamics_penalty_std | 0.0153 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 1800000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9285 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9483 +num rollout transitions: 250000, reward mean: 8.9674 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9771 +num rollout transitions: 250000, reward mean: 8.9618 +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9616 +num rollout transitions: 250000, reward mean: 8.9324 +num rollout transitions: 250000, reward mean: 8.9374 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9551 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9195 +num rollout transitions: 250000, reward mean: 8.9677 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9380 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9043 +num rollout transitions: 250000, reward mean: 8.9416 +num rollout transitions: 250000, reward mean: 8.9686 +num rollout transitions: 250000, reward mean: 8.9472 +num rollout transitions: 250000, reward mean: 8.9750 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9218 +num rollout transitions: 250000, reward mean: 8.9401 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9527 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9268 +num rollout transitions: 250000, reward mean: 8.9631 +num rollout transitions: 250000, reward mean: 8.9119 +num rollout transitions: 250000, reward mean: 8.9328 +num rollout transitions: 250000, reward mean: 8.9184 +num rollout transitions: 250000, reward mean: 8.9467 +num rollout transitions: 250000, reward mean: 8.9651 +num rollout transitions: 250000, reward mean: 8.9248 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.9266 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9315 +num rollout transitions: 250000, reward mean: 8.9647 +num rollout transitions: 250000, reward mean: 8.9194 +num rollout transitions: 250000, reward mean: 8.9287 +---------------------------------------------------------------------------------- +| alpha | 0.344 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 3.11 | +| loss/actor | -992 | +| loss/alpha | 0.0016 | +| loss/critic | 74.8 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.67 | +| rollout_info/dynamics_penalty_mean | 0.00025 | +| rollout_info/dynamics_penalty_std | 0.015 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9426 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9450 +num rollout transitions: 250000, reward mean: 8.9239 +num rollout transitions: 250000, reward mean: 8.9114 +num rollout transitions: 250000, reward mean: 8.9401 +num rollout transitions: 250000, reward mean: 8.9555 +num rollout transitions: 250000, reward mean: 8.9113 +num rollout transitions: 250000, reward mean: 8.9181 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9636 +num rollout transitions: 250000, reward mean: 8.9507 +num rollout transitions: 250000, reward mean: 8.9426 +num rollout transitions: 250000, reward mean: 8.9469 +num rollout transitions: 250000, reward mean: 8.9498 +num rollout transitions: 250000, reward mean: 8.9575 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9480 +num rollout transitions: 250000, reward mean: 8.9252 +num rollout transitions: 250000, reward mean: 8.9455 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9508 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9289 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9751 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9609 +num rollout transitions: 250000, reward mean: 8.9566 +num rollout transitions: 250000, reward mean: 8.9325 +num rollout transitions: 250000, reward mean: 8.9839 +num rollout transitions: 250000, reward mean: 8.9578 +num rollout transitions: 250000, reward mean: 8.9654 +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9654 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9347 +num rollout transitions: 250000, reward mean: 8.9252 +num rollout transitions: 250000, reward mean: 8.9675 +num rollout transitions: 250000, reward mean: 8.9332 +num rollout transitions: 250000, reward mean: 8.9364 +num rollout transitions: 250000, reward mean: 8.9495 +---------------------------------------------------------------------------------- +| alpha | 0.347 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 14.8 | +| loss/actor | -997 | +| loss/alpha | -0.00102 | +| loss/critic | 72.1 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0.000266 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9535 +num rollout transitions: 250000, reward mean: 8.9874 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.9493 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9255 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9195 +num rollout transitions: 250000, reward mean: 8.9310 +num rollout transitions: 250000, reward mean: 8.9482 +num rollout transitions: 250000, reward mean: 8.9683 +num rollout transitions: 250000, reward mean: 8.9635 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9390 +num rollout transitions: 250000, reward mean: 8.9283 +num rollout transitions: 250000, reward mean: 8.9250 +num rollout transitions: 250000, reward mean: 8.9387 +num rollout transitions: 250000, reward mean: 8.9627 +num rollout transitions: 250000, reward mean: 8.9344 +num rollout transitions: 250000, reward mean: 8.9472 +num rollout transitions: 250000, reward mean: 8.9244 +num rollout transitions: 250000, reward mean: 8.9487 +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.9298 +num rollout transitions: 250000, reward mean: 8.9333 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9688 +num rollout transitions: 250000, reward mean: 8.8931 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.8986 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9161 +num rollout transitions: 250000, reward mean: 8.9305 +num rollout transitions: 250000, reward mean: 8.9439 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9354 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9680 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9269 +num rollout transitions: 250000, reward mean: 8.9665 +num rollout transitions: 250000, reward mean: 8.9288 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9352 +num rollout transitions: 250000, reward mean: 8.9712 +num rollout transitions: 250000, reward mean: 8.9504 +num rollout transitions: 250000, reward mean: 8.9650 +num rollout transitions: 250000, reward mean: 8.9443 +---------------------------------------------------------------------------------- +| alpha | 0.345 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 5.11 | +| loss/actor | -998 | +| loss/alpha | -0.00115 | +| loss/critic | 75.4 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0.00026 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9505 +num rollout transitions: 250000, reward mean: 8.9526 +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.8909 +num rollout transitions: 250000, reward mean: 8.9595 +num rollout transitions: 250000, reward mean: 8.9269 +num rollout transitions: 250000, reward mean: 8.9040 +num rollout transitions: 250000, reward mean: 8.9641 +num rollout transitions: 250000, reward mean: 8.9494 +num rollout transitions: 250000, reward mean: 8.9717 +num rollout transitions: 250000, reward mean: 8.9549 +num rollout transitions: 250000, reward mean: 8.9214 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9557 +num rollout transitions: 250000, reward mean: 8.9532 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9357 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9459 +num rollout transitions: 250000, reward mean: 8.9296 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9102 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.9342 +num rollout transitions: 250000, reward mean: 8.9401 +num rollout transitions: 250000, reward mean: 8.9582 +num rollout transitions: 250000, reward mean: 8.9376 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9142 +num rollout transitions: 250000, reward mean: 8.9552 +num rollout transitions: 250000, reward mean: 8.9616 +num rollout transitions: 250000, reward mean: 8.9296 +num rollout transitions: 250000, reward mean: 8.9844 +num rollout transitions: 250000, reward mean: 8.9659 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9596 +num rollout transitions: 250000, reward mean: 8.9650 +num rollout transitions: 250000, reward mean: 8.9502 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9136 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9625 +num rollout transitions: 250000, reward mean: 8.9047 +num rollout transitions: 250000, reward mean: 8.9670 +---------------------------------------------------------------------------------- +| alpha | 0.344 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 5.13 | +| loss/actor | -998 | +| loss/alpha | 0.00155 | +| loss/critic | 72.2 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0.00026 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9368 +num rollout transitions: 250000, reward mean: 8.9571 +num rollout transitions: 250000, reward mean: 8.9493 +num rollout transitions: 250000, reward mean: 8.9246 +num rollout transitions: 250000, reward mean: 8.9253 +num rollout transitions: 250000, reward mean: 8.9416 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 8.9347 +num rollout transitions: 250000, reward mean: 8.9650 +num rollout transitions: 250000, reward mean: 8.9714 +num rollout transitions: 250000, reward mean: 8.9518 +num rollout transitions: 250000, reward mean: 8.9133 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9240 +num rollout transitions: 250000, reward mean: 8.9452 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9252 +num rollout transitions: 250000, reward mean: 8.9261 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9783 +num rollout transitions: 250000, reward mean: 8.9390 +num rollout transitions: 250000, reward mean: 8.9313 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9744 +num rollout transitions: 250000, reward mean: 8.9115 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9466 +num rollout transitions: 250000, reward mean: 8.9512 +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9328 +num rollout transitions: 250000, reward mean: 8.9762 +num rollout transitions: 250000, reward mean: 8.9463 +num rollout transitions: 250000, reward mean: 8.9566 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9605 +num rollout transitions: 250000, reward mean: 8.9273 +num rollout transitions: 250000, reward mean: 8.9349 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9811 +num rollout transitions: 250000, reward mean: 8.9373 +num rollout transitions: 250000, reward mean: 8.8970 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9503 +num rollout transitions: 250000, reward mean: 8.9355 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9411 +----------------------------------------------------------------------------------- +| alpha | 0.343 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 7.17 | +| loss/actor | -998 | +| loss/alpha | -0.000766 | +| loss/critic | 73.7 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0.000257 | +| rollout_info/dynamics_penalty_std | 0.0153 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 2050000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9178 +num rollout transitions: 250000, reward mean: 8.9510 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.9622 +num rollout transitions: 250000, reward mean: 8.9188 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.9508 +num rollout transitions: 250000, reward mean: 8.9498 +num rollout transitions: 250000, reward mean: 8.9613 +num rollout transitions: 250000, reward mean: 8.9664 +num rollout transitions: 250000, reward mean: 8.9695 +num rollout transitions: 250000, reward mean: 8.9160 +num rollout transitions: 250000, reward mean: 8.8994 +num rollout transitions: 250000, reward mean: 8.9852 +num rollout transitions: 250000, reward mean: 8.9208 +num rollout transitions: 250000, reward mean: 8.9326 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9528 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9742 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.9371 +num rollout transitions: 250000, reward mean: 8.9456 +num rollout transitions: 250000, reward mean: 8.9315 +num rollout transitions: 250000, reward mean: 8.9243 +num rollout transitions: 250000, reward mean: 8.9598 +num rollout transitions: 250000, reward mean: 8.9242 +num rollout transitions: 250000, reward mean: 8.9146 +num rollout transitions: 250000, reward mean: 8.9220 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9429 +num rollout transitions: 250000, reward mean: 8.9575 +num rollout transitions: 250000, reward mean: 8.9305 +num rollout transitions: 250000, reward mean: 8.9055 +num rollout transitions: 250000, reward mean: 8.9345 +num rollout transitions: 250000, reward mean: 8.9470 +num rollout transitions: 250000, reward mean: 8.9141 +num rollout transitions: 250000, reward mean: 8.9642 +num rollout transitions: 250000, reward mean: 8.9151 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9036 +num rollout transitions: 250000, reward mean: 8.9381 +num rollout transitions: 250000, reward mean: 8.9510 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9364 +num rollout transitions: 250000, reward mean: 8.9637 +num rollout transitions: 250000, reward mean: 8.9558 +num rollout transitions: 250000, reward mean: 8.9464 +---------------------------------------------------------------------------------- +| alpha | 0.341 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 98.1 | +| eval/normalized_episode_reward_std | 9.65 | +| loss/actor | -998 | +| loss/alpha | -0.00061 | +| loss/critic | 73.1 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.000265 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9535 +num rollout transitions: 250000, reward mean: 8.9212 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9897 +num rollout transitions: 250000, reward mean: 8.9261 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9395 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9528 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9420 +num rollout transitions: 250000, reward mean: 8.9583 +num rollout transitions: 250000, reward mean: 8.9426 +num rollout transitions: 250000, reward mean: 8.9768 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9285 +num rollout transitions: 250000, reward mean: 8.9332 +num rollout transitions: 250000, reward mean: 8.9672 +num rollout transitions: 250000, reward mean: 8.9296 +num rollout transitions: 250000, reward mean: 8.9459 +num rollout transitions: 250000, reward mean: 8.9213 +num rollout transitions: 250000, reward mean: 8.9593 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9598 +num rollout transitions: 250000, reward mean: 8.9288 +num rollout transitions: 250000, reward mean: 8.9200 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9585 +num rollout transitions: 250000, reward mean: 8.9581 +num rollout transitions: 250000, reward mean: 8.9188 +num rollout transitions: 250000, reward mean: 8.9885 +num rollout transitions: 250000, reward mean: 8.9338 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9186 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9517 +num rollout transitions: 250000, reward mean: 8.9155 +num rollout transitions: 250000, reward mean: 8.9247 +num rollout transitions: 250000, reward mean: 8.9558 +num rollout transitions: 250000, reward mean: 8.9710 +num rollout transitions: 250000, reward mean: 8.9436 +num rollout transitions: 250000, reward mean: 8.9489 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.9281 +num rollout transitions: 250000, reward mean: 8.9369 +num rollout transitions: 250000, reward mean: 8.9466 +num rollout transitions: 250000, reward mean: 8.9222 +num rollout transitions: 250000, reward mean: 8.9501 +num rollout transitions: 250000, reward mean: 8.9578 +---------------------------------------------------------------------------------- +| alpha | 0.339 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 7.72 | +| loss/actor | -1e+03 | +| loss/alpha | 0.00173 | +| loss/critic | 67.4 | +| penalty/lcb_mean | 1.56 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0.000264 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9533 +num rollout transitions: 250000, reward mean: 8.9703 +num rollout transitions: 250000, reward mean: 8.9552 +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9222 +num rollout transitions: 250000, reward mean: 8.9602 +num rollout transitions: 250000, reward mean: 8.9455 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9429 +num rollout transitions: 250000, reward mean: 8.9724 +num rollout transitions: 250000, reward mean: 8.9426 +num rollout transitions: 250000, reward mean: 8.9198 +num rollout transitions: 250000, reward mean: 8.9590 +num rollout transitions: 250000, reward mean: 8.9596 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9298 +num rollout transitions: 250000, reward mean: 8.9606 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9497 +num rollout transitions: 250000, reward mean: 8.9518 +num rollout transitions: 250000, reward mean: 8.9661 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9338 +num rollout transitions: 250000, reward mean: 8.9653 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9326 +num rollout transitions: 250000, reward mean: 8.9602 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9633 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9601 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9328 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9575 +num rollout transitions: 250000, reward mean: 8.9631 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9506 +num rollout transitions: 250000, reward mean: 8.9199 +num rollout transitions: 250000, reward mean: 8.9684 +----------------------------------------------------------------------------------- +| alpha | 0.342 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 3.13 | +| loss/actor | -1.01e+03 | +| loss/alpha | -0.0017 | +| loss/critic | 72.2 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.000255 | +| rollout_info/dynamics_penalty_std | 0.0152 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2200000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9518 +num rollout transitions: 250000, reward mean: 8.9312 +num rollout transitions: 250000, reward mean: 8.9148 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9480 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9781 +num rollout transitions: 250000, reward mean: 8.9652 +num rollout transitions: 250000, reward mean: 8.9612 +num rollout transitions: 250000, reward mean: 8.9848 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9597 +num rollout transitions: 250000, reward mean: 8.9463 +num rollout transitions: 250000, reward mean: 8.9477 +num rollout transitions: 250000, reward mean: 8.9502 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9379 +num rollout transitions: 250000, reward mean: 8.9523 +num rollout transitions: 250000, reward mean: 8.9652 +num rollout transitions: 250000, reward mean: 8.9487 +num rollout transitions: 250000, reward mean: 8.9845 +num rollout transitions: 250000, reward mean: 8.9413 +num rollout transitions: 250000, reward mean: 8.9468 +num rollout transitions: 250000, reward mean: 8.9219 +num rollout transitions: 250000, reward mean: 8.9336 +num rollout transitions: 250000, reward mean: 8.9543 +num rollout transitions: 250000, reward mean: 8.9580 +num rollout transitions: 250000, reward mean: 8.9295 +num rollout transitions: 250000, reward mean: 8.9442 +num rollout transitions: 250000, reward mean: 8.9594 +num rollout transitions: 250000, reward mean: 8.9335 +num rollout transitions: 250000, reward mean: 8.9100 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9652 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9322 +num rollout transitions: 250000, reward mean: 8.9285 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9640 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9337 +num rollout transitions: 250000, reward mean: 8.9449 +num rollout transitions: 250000, reward mean: 8.9712 +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9667 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9609 +----------------------------------------------------------------------------------- +| alpha | 0.343 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 3.22 | +| loss/actor | -1.01e+03 | +| loss/alpha | 0.00121 | +| loss/critic | 73 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0.00026 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2250000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9393 +num rollout transitions: 250000, reward mean: 8.9477 +num rollout transitions: 250000, reward mean: 8.9514 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9440 +num rollout transitions: 250000, reward mean: 8.9521 +num rollout transitions: 250000, reward mean: 8.9613 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9481 +num rollout transitions: 250000, reward mean: 8.9708 +num rollout transitions: 250000, reward mean: 8.9421 +num rollout transitions: 250000, reward mean: 8.9364 +num rollout transitions: 250000, reward mean: 8.9219 +num rollout transitions: 250000, reward mean: 8.9612 +num rollout transitions: 250000, reward mean: 8.9419 +num rollout transitions: 250000, reward mean: 8.9498 +num rollout transitions: 250000, reward mean: 8.9426 +num rollout transitions: 250000, reward mean: 8.9637 +num rollout transitions: 250000, reward mean: 8.9506 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9605 +num rollout transitions: 250000, reward mean: 8.9752 +num rollout transitions: 250000, reward mean: 8.9467 +num rollout transitions: 250000, reward mean: 8.9442 +num rollout transitions: 250000, reward mean: 8.9672 +num rollout transitions: 250000, reward mean: 8.9294 +num rollout transitions: 250000, reward mean: 8.9649 +num rollout transitions: 250000, reward mean: 8.9191 +num rollout transitions: 250000, reward mean: 8.9720 +num rollout transitions: 250000, reward mean: 8.9498 +num rollout transitions: 250000, reward mean: 8.9207 +num rollout transitions: 250000, reward mean: 8.9528 +num rollout transitions: 250000, reward mean: 8.9602 +num rollout transitions: 250000, reward mean: 8.9365 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9617 +num rollout transitions: 250000, reward mean: 8.9511 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9396 +num rollout transitions: 250000, reward mean: 8.9336 +num rollout transitions: 250000, reward mean: 8.9634 +num rollout transitions: 250000, reward mean: 8.9388 +num rollout transitions: 250000, reward mean: 8.9502 +num rollout transitions: 250000, reward mean: 8.9415 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9460 +----------------------------------------------------------------------------------- +| alpha | 0.342 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 9.04 | +| loss/actor | -1.01e+03 | +| loss/alpha | -0.00122 | +| loss/critic | 74.1 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.69 | +| rollout_info/dynamics_penalty_mean | 0.000267 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2300000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9427 +num rollout transitions: 250000, reward mean: 8.9364 +num rollout transitions: 250000, reward mean: 8.9580 +num rollout transitions: 250000, reward mean: 8.9549 +num rollout transitions: 250000, reward mean: 8.9347 +num rollout transitions: 250000, reward mean: 8.8994 +num rollout transitions: 250000, reward mean: 8.9697 +num rollout transitions: 250000, reward mean: 8.9335 +num rollout transitions: 250000, reward mean: 8.9584 +num rollout transitions: 250000, reward mean: 8.9328 +num rollout transitions: 250000, reward mean: 8.9300 +num rollout transitions: 250000, reward mean: 8.9687 +num rollout transitions: 250000, reward mean: 8.9507 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9479 +num rollout transitions: 250000, reward mean: 8.9189 +num rollout transitions: 250000, reward mean: 8.9445 +num rollout transitions: 250000, reward mean: 8.9092 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9668 +num rollout transitions: 250000, reward mean: 8.9138 +num rollout transitions: 250000, reward mean: 8.9210 +num rollout transitions: 250000, reward mean: 8.9468 +num rollout transitions: 250000, reward mean: 8.9220 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9483 +num rollout transitions: 250000, reward mean: 8.9194 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9507 +num rollout transitions: 250000, reward mean: 8.9383 +num rollout transitions: 250000, reward mean: 8.9564 +num rollout transitions: 250000, reward mean: 8.9656 +num rollout transitions: 250000, reward mean: 8.9474 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.9467 +num rollout transitions: 250000, reward mean: 8.9385 +num rollout transitions: 250000, reward mean: 8.9368 +num rollout transitions: 250000, reward mean: 8.9708 +num rollout transitions: 250000, reward mean: 8.9478 +num rollout transitions: 250000, reward mean: 8.9419 +num rollout transitions: 250000, reward mean: 8.9486 +num rollout transitions: 250000, reward mean: 8.9651 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.9247 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9503 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9456 +num rollout transitions: 250000, reward mean: 8.9306 +---------------------------------------------------------------------------------- +| alpha | 0.335 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 15.4 | +| loss/actor | -1e+03 | +| loss/alpha | 0.00144 | +| loss/critic | 74.9 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0.000253 | +| rollout_info/dynamics_penalty_std | 0.0151 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9325 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9619 +num rollout transitions: 250000, reward mean: 8.9767 +num rollout transitions: 250000, reward mean: 8.9413 +num rollout transitions: 250000, reward mean: 8.9198 +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.9461 +num rollout transitions: 250000, reward mean: 8.9537 +num rollout transitions: 250000, reward mean: 8.9269 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9417 +num rollout transitions: 250000, reward mean: 8.9477 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9607 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9731 +num rollout transitions: 250000, reward mean: 8.9457 +num rollout transitions: 250000, reward mean: 8.9562 +num rollout transitions: 250000, reward mean: 8.9708 +num rollout transitions: 250000, reward mean: 8.9379 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9184 +num rollout transitions: 250000, reward mean: 8.9699 +num rollout transitions: 250000, reward mean: 8.9885 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9753 +num rollout transitions: 250000, reward mean: 8.9736 +num rollout transitions: 250000, reward mean: 8.9643 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9307 +num rollout transitions: 250000, reward mean: 8.9360 +num rollout transitions: 250000, reward mean: 8.9508 +num rollout transitions: 250000, reward mean: 8.9563 +num rollout transitions: 250000, reward mean: 8.9558 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9140 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9571 +num rollout transitions: 250000, reward mean: 8.9487 +num rollout transitions: 250000, reward mean: 8.9463 +num rollout transitions: 250000, reward mean: 8.9662 +num rollout transitions: 250000, reward mean: 8.9369 +num rollout transitions: 250000, reward mean: 8.9578 +num rollout transitions: 250000, reward mean: 8.9374 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9257 +----------------------------------------------------------------------------------- +| alpha | 0.342 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.6 | +| eval/normalized_episode_reward_std | 15.3 | +| loss/actor | -1.01e+03 | +| loss/alpha | 1.03e-05 | +| loss/critic | 72.3 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.67 | +| rollout_info/dynamics_penalty_mean | 0.000256 | +| rollout_info/dynamics_penalty_std | 0.0153 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2400000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9510 +num rollout transitions: 250000, reward mean: 8.9596 +num rollout transitions: 250000, reward mean: 8.9511 +num rollout transitions: 250000, reward mean: 8.9473 +num rollout transitions: 250000, reward mean: 8.9590 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9243 +num rollout transitions: 250000, reward mean: 8.9634 +num rollout transitions: 250000, reward mean: 8.9356 +num rollout transitions: 250000, reward mean: 8.9601 +num rollout transitions: 250000, reward mean: 8.9714 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9196 +num rollout transitions: 250000, reward mean: 8.9493 +num rollout transitions: 250000, reward mean: 8.9049 +num rollout transitions: 250000, reward mean: 8.9499 +num rollout transitions: 250000, reward mean: 8.9621 +num rollout transitions: 250000, reward mean: 8.9169 +num rollout transitions: 250000, reward mean: 8.9376 +num rollout transitions: 250000, reward mean: 8.9368 +num rollout transitions: 250000, reward mean: 8.9661 +num rollout transitions: 250000, reward mean: 8.9453 +num rollout transitions: 250000, reward mean: 8.9329 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9414 +num rollout transitions: 250000, reward mean: 8.8999 +num rollout transitions: 250000, reward mean: 8.9355 +num rollout transitions: 250000, reward mean: 8.9321 +num rollout transitions: 250000, reward mean: 8.9642 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9501 +num rollout transitions: 250000, reward mean: 8.9395 +num rollout transitions: 250000, reward mean: 8.9494 +num rollout transitions: 250000, reward mean: 8.9703 +num rollout transitions: 250000, reward mean: 8.9595 +num rollout transitions: 250000, reward mean: 8.9265 +num rollout transitions: 250000, reward mean: 8.9304 +num rollout transitions: 250000, reward mean: 8.9354 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9352 +num rollout transitions: 250000, reward mean: 8.9554 +num rollout transitions: 250000, reward mean: 8.9348 +num rollout transitions: 250000, reward mean: 8.9403 +num rollout transitions: 250000, reward mean: 8.9444 +num rollout transitions: 250000, reward mean: 8.9689 +num rollout transitions: 250000, reward mean: 8.9472 +num rollout transitions: 250000, reward mean: 8.9626 +num rollout transitions: 250000, reward mean: 8.9538 +num rollout transitions: 250000, reward mean: 8.9440 +----------------------------------------------------------------------------------- +| alpha | 0.341 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 3.2 | +| loss/actor | -1.01e+03 | +| loss/alpha | -0.000733 | +| loss/critic | 75.4 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.7 | +| rollout_info/dynamics_penalty_mean | 0.000268 | +| rollout_info/dynamics_penalty_std | 0.0156 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.94 | +| timestep | 2450000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9593 +num rollout transitions: 250000, reward mean: 8.9712 +num rollout transitions: 250000, reward mean: 8.9600 +num rollout transitions: 250000, reward mean: 8.9307 +num rollout transitions: 250000, reward mean: 8.9264 +num rollout transitions: 250000, reward mean: 8.9712 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9472 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9274 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9616 +num rollout transitions: 250000, reward mean: 8.9806 +num rollout transitions: 250000, reward mean: 8.9447 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9514 +num rollout transitions: 250000, reward mean: 8.9409 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9403 +num rollout transitions: 250000, reward mean: 8.9413 +num rollout transitions: 250000, reward mean: 8.9600 +num rollout transitions: 250000, reward mean: 8.9734 +num rollout transitions: 250000, reward mean: 8.9330 +num rollout transitions: 250000, reward mean: 8.9697 +num rollout transitions: 250000, reward mean: 8.9802 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9377 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9556 +num rollout transitions: 250000, reward mean: 8.9354 +num rollout transitions: 250000, reward mean: 8.9302 +num rollout transitions: 250000, reward mean: 8.9773 +num rollout transitions: 250000, reward mean: 8.9615 +num rollout transitions: 250000, reward mean: 8.9757 +num rollout transitions: 250000, reward mean: 8.9569 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9479 +num rollout transitions: 250000, reward mean: 8.9333 +num rollout transitions: 250000, reward mean: 8.9732 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9685 +num rollout transitions: 250000, reward mean: 8.9432 +num rollout transitions: 250000, reward mean: 8.9585 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9253 +num rollout transitions: 250000, reward mean: 8.9601 +num rollout transitions: 250000, reward mean: 8.9593 +num rollout transitions: 250000, reward mean: 8.9659 +num rollout transitions: 250000, reward mean: 8.9421 +num rollout transitions: 250000, reward mean: 8.9605 +----------------------------------------------------------------------------------- +| alpha | 0.341 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 7.49 | +| loss/actor | -1.01e+03 | +| loss/alpha | 0.000793 | +| loss/critic | 72.4 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0.000261 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2500000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9456 +num rollout transitions: 250000, reward mean: 8.9472 +num rollout transitions: 250000, reward mean: 8.9545 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9284 +num rollout transitions: 250000, reward mean: 8.9630 +num rollout transitions: 250000, reward mean: 8.9441 +num rollout transitions: 250000, reward mean: 8.9678 +num rollout transitions: 250000, reward mean: 8.9362 +num rollout transitions: 250000, reward mean: 8.9293 +num rollout transitions: 250000, reward mean: 8.9401 +num rollout transitions: 250000, reward mean: 8.9504 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9552 +num rollout transitions: 250000, reward mean: 8.9397 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9747 +num rollout transitions: 250000, reward mean: 8.9585 +num rollout transitions: 250000, reward mean: 8.9533 +num rollout transitions: 250000, reward mean: 8.9765 +num rollout transitions: 250000, reward mean: 8.9468 +num rollout transitions: 250000, reward mean: 8.9306 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.9320 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9799 +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9369 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9551 +num rollout transitions: 250000, reward mean: 8.9457 +num rollout transitions: 250000, reward mean: 8.9352 +num rollout transitions: 250000, reward mean: 8.9401 +num rollout transitions: 250000, reward mean: 8.9682 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 8.9564 +num rollout transitions: 250000, reward mean: 8.9360 +num rollout transitions: 250000, reward mean: 8.9750 +num rollout transitions: 250000, reward mean: 8.9375 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 8.9506 +num rollout transitions: 250000, reward mean: 8.9304 +num rollout transitions: 250000, reward mean: 8.9555 +num rollout transitions: 250000, reward mean: 8.9178 +num rollout transitions: 250000, reward mean: 8.9480 +num rollout transitions: 250000, reward mean: 8.9297 +----------------------------------------------------------------------------------- +| alpha | 0.343 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 9.62 | +| loss/actor | -1.01e+03 | +| loss/alpha | -0.000312 | +| loss/critic | 73.1 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0.000272 | +| rollout_info/dynamics_penalty_std | 0.0157 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2550000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9376 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9474 +num rollout transitions: 250000, reward mean: 8.9552 +num rollout transitions: 250000, reward mean: 8.9253 +num rollout transitions: 250000, reward mean: 8.9571 +num rollout transitions: 250000, reward mean: 8.9481 +num rollout transitions: 250000, reward mean: 8.9329 +num rollout transitions: 250000, reward mean: 8.9400 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9481 +num rollout transitions: 250000, reward mean: 8.9358 +num rollout transitions: 250000, reward mean: 8.9464 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9382 +num rollout transitions: 250000, reward mean: 8.9587 +num rollout transitions: 250000, reward mean: 8.9613 +num rollout transitions: 250000, reward mean: 8.9417 +num rollout transitions: 250000, reward mean: 8.9766 +num rollout transitions: 250000, reward mean: 8.9430 +num rollout transitions: 250000, reward mean: 8.9664 +num rollout transitions: 250000, reward mean: 8.9611 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9512 +num rollout transitions: 250000, reward mean: 8.9434 +num rollout transitions: 250000, reward mean: 8.9537 +num rollout transitions: 250000, reward mean: 8.9368 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9694 +num rollout transitions: 250000, reward mean: 8.9634 +num rollout transitions: 250000, reward mean: 8.9364 +num rollout transitions: 250000, reward mean: 8.9511 +num rollout transitions: 250000, reward mean: 8.9595 +num rollout transitions: 250000, reward mean: 8.9405 +num rollout transitions: 250000, reward mean: 8.9341 +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9832 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9697 +num rollout transitions: 250000, reward mean: 8.9961 +num rollout transitions: 250000, reward mean: 8.9594 +num rollout transitions: 250000, reward mean: 8.9365 +num rollout transitions: 250000, reward mean: 8.9640 +num rollout transitions: 250000, reward mean: 8.9455 +num rollout transitions: 250000, reward mean: 8.9504 +num rollout transitions: 250000, reward mean: 8.9542 +num rollout transitions: 250000, reward mean: 8.9485 +num rollout transitions: 250000, reward mean: 8.9506 +----------------------------------------------------------------------------------- +| alpha | 0.343 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 14.9 | +| loss/actor | -1.01e+03 | +| loss/alpha | 0.00178 | +| loss/critic | 73.3 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.64 | +| rollout_info/dynamics_penalty_mean | 0.00026 | +| rollout_info/dynamics_penalty_std | 0.0153 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2600000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9396 +num rollout transitions: 250000, reward mean: 8.9768 +num rollout transitions: 250000, reward mean: 8.9326 +num rollout transitions: 250000, reward mean: 8.9566 +num rollout transitions: 250000, reward mean: 8.9519 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9527 +num rollout transitions: 250000, reward mean: 8.9606 +num rollout transitions: 250000, reward mean: 8.9336 +num rollout transitions: 250000, reward mean: 8.9656 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9671 +num rollout transitions: 250000, reward mean: 8.9557 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9402 +num rollout transitions: 250000, reward mean: 8.9310 +num rollout transitions: 250000, reward mean: 8.9497 +num rollout transitions: 250000, reward mean: 8.9353 +num rollout transitions: 250000, reward mean: 8.9600 +num rollout transitions: 250000, reward mean: 8.9565 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9574 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9728 +num rollout transitions: 250000, reward mean: 8.9333 +num rollout transitions: 250000, reward mean: 8.9313 +num rollout transitions: 250000, reward mean: 8.9669 +num rollout transitions: 250000, reward mean: 8.9725 +num rollout transitions: 250000, reward mean: 8.9351 +num rollout transitions: 250000, reward mean: 8.9819 +num rollout transitions: 250000, reward mean: 8.9172 +num rollout transitions: 250000, reward mean: 8.9646 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9665 +num rollout transitions: 250000, reward mean: 8.9824 +num rollout transitions: 250000, reward mean: 8.9350 +num rollout transitions: 250000, reward mean: 8.9209 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9353 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9418 +num rollout transitions: 250000, reward mean: 8.9516 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9189 +num rollout transitions: 250000, reward mean: 8.9172 +num rollout transitions: 250000, reward mean: 8.9597 +num rollout transitions: 250000, reward mean: 8.9571 +num rollout transitions: 250000, reward mean: 8.9327 +num rollout transitions: 250000, reward mean: 8.9530 +----------------------------------------------------------------------------------- +| alpha | 0.341 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 7.34 | +| loss/actor | -1.01e+03 | +| loss/alpha | -0.00255 | +| loss/critic | 75.1 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.7 | +| rollout_info/dynamics_penalty_mean | 0.000248 | +| rollout_info/dynamics_penalty_std | 0.0149 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2650000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9611 +num rollout transitions: 250000, reward mean: 8.9486 +num rollout transitions: 250000, reward mean: 8.9607 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9672 +num rollout transitions: 250000, reward mean: 8.9518 +num rollout transitions: 250000, reward mean: 8.9454 +num rollout transitions: 250000, reward mean: 8.9504 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9530 +num rollout transitions: 250000, reward mean: 8.9311 +num rollout transitions: 250000, reward mean: 8.9294 +num rollout transitions: 250000, reward mean: 8.9728 +num rollout transitions: 250000, reward mean: 8.9248 +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9581 +num rollout transitions: 250000, reward mean: 8.9657 +num rollout transitions: 250000, reward mean: 8.9627 +num rollout transitions: 250000, reward mean: 8.9262 +num rollout transitions: 250000, reward mean: 8.9509 +num rollout transitions: 250000, reward mean: 8.9159 +num rollout transitions: 250000, reward mean: 8.9528 +num rollout transitions: 250000, reward mean: 8.9563 +num rollout transitions: 250000, reward mean: 8.9549 +num rollout transitions: 250000, reward mean: 8.9345 +num rollout transitions: 250000, reward mean: 8.9678 +num rollout transitions: 250000, reward mean: 8.9477 +num rollout transitions: 250000, reward mean: 8.9518 +num rollout transitions: 250000, reward mean: 8.9412 +num rollout transitions: 250000, reward mean: 8.9242 +num rollout transitions: 250000, reward mean: 8.9703 +num rollout transitions: 250000, reward mean: 8.9839 +num rollout transitions: 250000, reward mean: 8.9113 +num rollout transitions: 250000, reward mean: 8.9417 +num rollout transitions: 250000, reward mean: 8.9352 +num rollout transitions: 250000, reward mean: 8.9458 +num rollout transitions: 250000, reward mean: 8.9508 +num rollout transitions: 250000, reward mean: 8.9553 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9591 +num rollout transitions: 250000, reward mean: 8.9555 +num rollout transitions: 250000, reward mean: 8.9608 +num rollout transitions: 250000, reward mean: 8.9352 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9600 +num rollout transitions: 250000, reward mean: 8.9642 +num rollout transitions: 250000, reward mean: 8.9302 +num rollout transitions: 250000, reward mean: 8.9240 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9386 +----------------------------------------------------------------------------------- +| alpha | 0.34 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 7.03 | +| loss/actor | -1.01e+03 | +| loss/alpha | 0.00213 | +| loss/critic | 73.4 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.69 | +| rollout_info/dynamics_penalty_mean | 0.000262 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2700000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9560 +num rollout transitions: 250000, reward mean: 8.9270 +num rollout transitions: 250000, reward mean: 8.9271 +num rollout transitions: 250000, reward mean: 8.9614 +num rollout transitions: 250000, reward mean: 8.9670 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9386 +num rollout transitions: 250000, reward mean: 8.9511 +num rollout transitions: 250000, reward mean: 8.9333 +num rollout transitions: 250000, reward mean: 8.9550 +num rollout transitions: 250000, reward mean: 8.9376 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9812 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9618 +num rollout transitions: 250000, reward mean: 8.9541 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9370 +num rollout transitions: 250000, reward mean: 8.9596 +num rollout transitions: 250000, reward mean: 8.9608 +num rollout transitions: 250000, reward mean: 8.9520 +num rollout transitions: 250000, reward mean: 8.9457 +num rollout transitions: 250000, reward mean: 8.9391 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9191 +num rollout transitions: 250000, reward mean: 8.9678 +num rollout transitions: 250000, reward mean: 8.9645 +num rollout transitions: 250000, reward mean: 8.9432 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.9638 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9343 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9486 +num rollout transitions: 250000, reward mean: 8.9536 +num rollout transitions: 250000, reward mean: 8.9549 +num rollout transitions: 250000, reward mean: 8.9638 +num rollout transitions: 250000, reward mean: 8.9570 +num rollout transitions: 250000, reward mean: 8.9332 +num rollout transitions: 250000, reward mean: 8.9346 +num rollout transitions: 250000, reward mean: 8.9691 +num rollout transitions: 250000, reward mean: 8.9838 +num rollout transitions: 250000, reward mean: 8.9480 +num rollout transitions: 250000, reward mean: 8.9544 +num rollout transitions: 250000, reward mean: 8.9520 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9496 +num rollout transitions: 250000, reward mean: 8.9422 +----------------------------------------------------------------------------------- +| alpha | 0.339 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 7.99 | +| loss/actor | -1.02e+03 | +| loss/alpha | -0.00199 | +| loss/critic | 74.4 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.73 | +| rollout_info/dynamics_penalty_mean | 0.000274 | +| rollout_info/dynamics_penalty_std | 0.0158 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2750000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9542 +num rollout transitions: 250000, reward mean: 8.9414 +num rollout transitions: 250000, reward mean: 8.9442 +num rollout transitions: 250000, reward mean: 8.9319 +num rollout transitions: 250000, reward mean: 8.9352 +num rollout transitions: 250000, reward mean: 8.9818 +num rollout transitions: 250000, reward mean: 8.9600 +num rollout transitions: 250000, reward mean: 8.9466 +num rollout transitions: 250000, reward mean: 8.9711 +num rollout transitions: 250000, reward mean: 8.9253 +num rollout transitions: 250000, reward mean: 8.9352 +num rollout transitions: 250000, reward mean: 8.9595 +num rollout transitions: 250000, reward mean: 8.9497 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9596 +num rollout transitions: 250000, reward mean: 8.9586 +num rollout transitions: 250000, reward mean: 8.9270 +num rollout transitions: 250000, reward mean: 8.9512 +num rollout transitions: 250000, reward mean: 8.9605 +num rollout transitions: 250000, reward mean: 8.9507 +num rollout transitions: 250000, reward mean: 8.9373 +num rollout transitions: 250000, reward mean: 8.9764 +num rollout transitions: 250000, reward mean: 8.9426 +num rollout transitions: 250000, reward mean: 8.9310 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9565 +num rollout transitions: 250000, reward mean: 8.9233 +num rollout transitions: 250000, reward mean: 8.9700 +num rollout transitions: 250000, reward mean: 8.9673 +num rollout transitions: 250000, reward mean: 8.9494 +num rollout transitions: 250000, reward mean: 8.9755 +num rollout transitions: 250000, reward mean: 8.9462 +num rollout transitions: 250000, reward mean: 8.9718 +num rollout transitions: 250000, reward mean: 8.9425 +num rollout transitions: 250000, reward mean: 8.9206 +num rollout transitions: 250000, reward mean: 8.9494 +num rollout transitions: 250000, reward mean: 8.9417 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9507 +num rollout transitions: 250000, reward mean: 8.9467 +num rollout transitions: 250000, reward mean: 8.9361 +num rollout transitions: 250000, reward mean: 8.9656 +num rollout transitions: 250000, reward mean: 8.9309 +num rollout transitions: 250000, reward mean: 8.9743 +num rollout transitions: 250000, reward mean: 8.9221 +num rollout transitions: 250000, reward mean: 8.9174 +num rollout transitions: 250000, reward mean: 8.9424 +num rollout transitions: 250000, reward mean: 8.9508 +num rollout transitions: 250000, reward mean: 8.9568 +----------------------------------------------------------------------------------- +| alpha | 0.341 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 10.6 | +| loss/actor | -1.02e+03 | +| loss/alpha | 0.000545 | +| loss/critic | 74.3 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.7 | +| rollout_info/dynamics_penalty_mean | 0.000261 | +| rollout_info/dynamics_penalty_std | 0.0153 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2800000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9667 +num rollout transitions: 250000, reward mean: 8.9436 +num rollout transitions: 250000, reward mean: 8.9647 +num rollout transitions: 250000, reward mean: 8.9548 +num rollout transitions: 250000, reward mean: 8.9364 +num rollout transitions: 250000, reward mean: 8.9308 +num rollout transitions: 250000, reward mean: 8.9457 +num rollout transitions: 250000, reward mean: 8.9621 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9419 +num rollout transitions: 250000, reward mean: 8.9641 +num rollout transitions: 250000, reward mean: 8.9394 +num rollout transitions: 250000, reward mean: 8.9460 +num rollout transitions: 250000, reward mean: 8.9197 +num rollout transitions: 250000, reward mean: 8.9692 +num rollout transitions: 250000, reward mean: 8.9312 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9364 +num rollout transitions: 250000, reward mean: 8.9617 +num rollout transitions: 250000, reward mean: 8.9309 +num rollout transitions: 250000, reward mean: 8.9414 +num rollout transitions: 250000, reward mean: 8.9235 +num rollout transitions: 250000, reward mean: 8.9363 +num rollout transitions: 250000, reward mean: 8.9521 +num rollout transitions: 250000, reward mean: 8.9726 +num rollout transitions: 250000, reward mean: 8.9491 +num rollout transitions: 250000, reward mean: 8.9299 +num rollout transitions: 250000, reward mean: 8.9629 +num rollout transitions: 250000, reward mean: 8.9649 +num rollout transitions: 250000, reward mean: 8.9378 +num rollout transitions: 250000, reward mean: 8.9765 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9435 +num rollout transitions: 250000, reward mean: 8.9406 +num rollout transitions: 250000, reward mean: 8.9493 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9413 +num rollout transitions: 250000, reward mean: 8.9614 +num rollout transitions: 250000, reward mean: 8.9469 +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9192 +num rollout transitions: 250000, reward mean: 8.9324 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9400 +num rollout transitions: 250000, reward mean: 8.9297 +num rollout transitions: 250000, reward mean: 8.9488 +num rollout transitions: 250000, reward mean: 8.9556 +num rollout transitions: 250000, reward mean: 8.9555 +num rollout transitions: 250000, reward mean: 8.9510 +num rollout transitions: 250000, reward mean: 8.9690 +----------------------------------------------------------------------------------- +| alpha | 0.342 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 6.92 | +| loss/actor | -1.02e+03 | +| loss/alpha | -0.00223 | +| loss/critic | 75.3 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.76 | +| rollout_info/dynamics_penalty_mean | 0.000271 | +| rollout_info/dynamics_penalty_std | 0.0157 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2850000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9546 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9567 +num rollout transitions: 250000, reward mean: 8.9255 +num rollout transitions: 250000, reward mean: 8.9583 +num rollout transitions: 250000, reward mean: 8.9509 +num rollout transitions: 250000, reward mean: 8.9623 +num rollout transitions: 250000, reward mean: 8.9660 +num rollout transitions: 250000, reward mean: 8.9477 +num rollout transitions: 250000, reward mean: 8.9307 +num rollout transitions: 250000, reward mean: 8.9622 +num rollout transitions: 250000, reward mean: 8.9698 +num rollout transitions: 250000, reward mean: 8.9568 +num rollout transitions: 250000, reward mean: 8.9407 +num rollout transitions: 250000, reward mean: 8.9719 +num rollout transitions: 250000, reward mean: 8.9410 +num rollout transitions: 250000, reward mean: 8.9642 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9750 +num rollout transitions: 250000, reward mean: 8.9531 +num rollout transitions: 250000, reward mean: 8.9707 +num rollout transitions: 250000, reward mean: 8.9448 +num rollout transitions: 250000, reward mean: 8.9484 +num rollout transitions: 250000, reward mean: 8.9551 +num rollout transitions: 250000, reward mean: 8.9501 +num rollout transitions: 250000, reward mean: 8.9396 +num rollout transitions: 250000, reward mean: 8.9603 +num rollout transitions: 250000, reward mean: 8.9340 +num rollout transitions: 250000, reward mean: 8.9627 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9582 +num rollout transitions: 250000, reward mean: 8.9938 +num rollout transitions: 250000, reward mean: 8.9677 +num rollout transitions: 250000, reward mean: 8.9315 +num rollout transitions: 250000, reward mean: 8.9690 +num rollout transitions: 250000, reward mean: 8.9492 +num rollout transitions: 250000, reward mean: 8.9599 +num rollout transitions: 250000, reward mean: 8.9334 +num rollout transitions: 250000, reward mean: 8.9666 +num rollout transitions: 250000, reward mean: 8.9503 +num rollout transitions: 250000, reward mean: 8.9540 +num rollout transitions: 250000, reward mean: 8.9469 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9441 +num rollout transitions: 250000, reward mean: 8.9506 +num rollout transitions: 250000, reward mean: 8.9572 +num rollout transitions: 250000, reward mean: 8.9693 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9610 +num rollout transitions: 250000, reward mean: 8.9340 +----------------------------------------------------------------------------------- +| alpha | 0.341 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 7.03 | +| loss/actor | -1.02e+03 | +| loss/alpha | 0.00439 | +| loss/critic | 76.5 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.74 | +| rollout_info/dynamics_penalty_mean | 0.000265 | +| rollout_info/dynamics_penalty_std | 0.0155 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2900000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9593 +num rollout transitions: 250000, reward mean: 8.9547 +num rollout transitions: 250000, reward mean: 8.9637 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.9483 +num rollout transitions: 250000, reward mean: 8.9981 +num rollout transitions: 250000, reward mean: 8.9582 +num rollout transitions: 250000, reward mean: 8.9332 +num rollout transitions: 250000, reward mean: 8.9372 +num rollout transitions: 250000, reward mean: 8.9515 +num rollout transitions: 250000, reward mean: 8.9676 +num rollout transitions: 250000, reward mean: 8.9528 +num rollout transitions: 250000, reward mean: 8.9616 +num rollout transitions: 250000, reward mean: 8.9359 +num rollout transitions: 250000, reward mean: 8.9179 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9542 +num rollout transitions: 250000, reward mean: 8.9381 +num rollout transitions: 250000, reward mean: 8.9429 +num rollout transitions: 250000, reward mean: 8.9542 +num rollout transitions: 250000, reward mean: 8.9431 +num rollout transitions: 250000, reward mean: 8.9675 +num rollout transitions: 250000, reward mean: 8.9387 +num rollout transitions: 250000, reward mean: 8.9626 +num rollout transitions: 250000, reward mean: 8.9667 +num rollout transitions: 250000, reward mean: 8.9511 +num rollout transitions: 250000, reward mean: 8.9646 +num rollout transitions: 250000, reward mean: 8.9700 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.9481 +num rollout transitions: 250000, reward mean: 8.9421 +num rollout transitions: 250000, reward mean: 8.9423 +num rollout transitions: 250000, reward mean: 8.9524 +num rollout transitions: 250000, reward mean: 8.9398 +num rollout transitions: 250000, reward mean: 8.9694 +num rollout transitions: 250000, reward mean: 8.9604 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9702 +num rollout transitions: 250000, reward mean: 8.9774 +num rollout transitions: 250000, reward mean: 8.9490 +num rollout transitions: 250000, reward mean: 8.9349 +num rollout transitions: 250000, reward mean: 8.9594 +num rollout transitions: 250000, reward mean: 8.9686 +num rollout transitions: 250000, reward mean: 8.9672 +num rollout transitions: 250000, reward mean: 8.9552 +num rollout transitions: 250000, reward mean: 8.9443 +num rollout transitions: 250000, reward mean: 8.9544 +num rollout transitions: 250000, reward mean: 8.9277 +num rollout transitions: 250000, reward mean: 8.9364 +----------------------------------------------------------------------------------- +| alpha | 0.344 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 3.15 | +| loss/actor | -1.02e+03 | +| loss/alpha | 0.000827 | +| loss/critic | 75.8 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.75 | +| rollout_info/dynamics_penalty_mean | 0.000263 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 2950000 | +----------------------------------------------------------------------------------- +num rollout transitions: 250000, reward mean: 8.9493 +num rollout transitions: 250000, reward mean: 8.9529 +num rollout transitions: 250000, reward mean: 8.9555 +num rollout transitions: 250000, reward mean: 8.9671 +num rollout transitions: 250000, reward mean: 8.9642 +num rollout transitions: 250000, reward mean: 8.9697 +num rollout transitions: 250000, reward mean: 8.9328 +num rollout transitions: 250000, reward mean: 8.9635 +num rollout transitions: 250000, reward mean: 8.9539 +num rollout transitions: 250000, reward mean: 8.9548 +num rollout transitions: 250000, reward mean: 8.9497 +num rollout transitions: 250000, reward mean: 8.9465 +num rollout transitions: 250000, reward mean: 8.9367 +num rollout transitions: 250000, reward mean: 8.9411 +num rollout transitions: 250000, reward mean: 8.9657 +num rollout transitions: 250000, reward mean: 8.9610 +num rollout transitions: 250000, reward mean: 8.9481 +num rollout transitions: 250000, reward mean: 8.9313 +num rollout transitions: 250000, reward mean: 8.9559 +num rollout transitions: 250000, reward mean: 8.9495 +num rollout transitions: 250000, reward mean: 8.9282 +num rollout transitions: 250000, reward mean: 8.9228 +num rollout transitions: 250000, reward mean: 8.9644 +num rollout transitions: 250000, reward mean: 8.9328 +num rollout transitions: 250000, reward mean: 8.9619 +num rollout transitions: 250000, reward mean: 8.9692 +num rollout transitions: 250000, reward mean: 8.9779 +num rollout transitions: 250000, reward mean: 8.9400 +num rollout transitions: 250000, reward mean: 8.9331 +num rollout transitions: 250000, reward mean: 8.9451 +num rollout transitions: 250000, reward mean: 8.9534 +num rollout transitions: 250000, reward mean: 8.9320 +num rollout transitions: 250000, reward mean: 8.9510 +num rollout transitions: 250000, reward mean: 8.9517 +num rollout transitions: 250000, reward mean: 8.9528 +num rollout transitions: 250000, reward mean: 8.9433 +num rollout transitions: 250000, reward mean: 8.9589 +num rollout transitions: 250000, reward mean: 8.9266 +num rollout transitions: 250000, reward mean: 8.9437 +num rollout transitions: 250000, reward mean: 8.9419 +num rollout transitions: 250000, reward mean: 8.9446 +num rollout transitions: 250000, reward mean: 8.9581 +num rollout transitions: 250000, reward mean: 8.9457 +num rollout transitions: 250000, reward mean: 8.9399 +num rollout transitions: 250000, reward mean: 8.9317 +num rollout transitions: 250000, reward mean: 8.9500 +num rollout transitions: 250000, reward mean: 8.9561 +num rollout transitions: 250000, reward mean: 8.9582 +num rollout transitions: 250000, reward mean: 8.9471 +num rollout transitions: 250000, reward mean: 8.9384 +----------------------------------------------------------------------------------- +| alpha | 0.337 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 3.1 | +| loss/actor | -1.02e+03 | +| loss/alpha | -0.00263 | +| loss/critic | 77 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.77 | +| rollout_info/dynamics_penalty_mean | 0.00026 | +| rollout_info/dynamics_penalty_std | 0.0154 | +| rollout_info/num_transitions | 2.5e+05 | +| rollout_info/reward_mean | 8.95 | +| timestep | 3000000 | +----------------------------------------------------------------------------------- +total time: 129854.67s diff --git a/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-001614/record/dynamics_training_progress.csv b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-001614/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-012638/model/mu.npy b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-012638/model/mu.npy new file mode 100644 index 0000000000000000000000000000000000000000..84e3f305629a6450584c210eff1b0aa62e6f1954 --- /dev/null +++ b/halfcheetah_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-012638/model/mu.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30bd2b9e32a28f3d98dd1ab014db7c4a9b2577454213dc1874a070d93fd9db6a +size 220 diff --git a/walker2d_medium_expert_sparse/diffusion_123/scheduler/scheduler_config.json b/walker2d_medium_expert_sparse/diffusion_123/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0dbe23fd6a0f8f13f0bce60db5ef70d55bec9257 --- /dev/null +++ b/walker2d_medium_expert_sparse/diffusion_123/scheduler/scheduler_config.json @@ -0,0 +1,19 @@ +{ + "_class_name": "DDIMScheduler", + "_diffusers_version": "0.36.0", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "clip_sample": true, + "clip_sample_range": 1.0, + "dynamic_thresholding_ratio": 0.995, + "num_train_timesteps": 50000, + "prediction_type": "epsilon", + "rescale_betas_zero_snr": false, + "sample_max_value": 1.0, + "set_alpha_to_one": true, + "steps_offset": 0, + "thresholding": false, + "timestep_spacing": "leading", + "trained_betas": null +} diff --git a/walker2d_medium_expert_sparse/diffusion_123/used_config.yaml b/walker2d_medium_expert_sparse/diffusion_123/used_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b263b542d24a33c1f39a95bdf1ca5c22078cebe --- /dev/null +++ b/walker2d_medium_expert_sparse/diffusion_123/used_config.yaml @@ -0,0 +1,21 @@ +npz: /public/d4rl/sparse_datasets/diffusion_processed/walker2d_medium_expert_sparse_73_train.npz +out: /public/gormpo/models/walker2d_medium_expert_sparse_3/diffusion +epochs: 100 +batch: 512 +lr: 0.0002 +wd: 0.0 +timesteps: 50000 +checkpoint_every_epochs: 10 +seed: 0 +device: cuda +model: mlp +hidden: 512 +time_emb: 128 +layers: 3 +dropout: 0.0 +wandb: false +wandb_project: GORMPO +wandb_entity: jhhuang_chloe +wandb_run: unconditional_ddim_mlp_walker2d_medium_expert_sparse_73 +log_every: 100 +samples_every: 1 diff --git a/walker2d_medium_expert_sparse/diffusion_42/scheduler/scheduler_config.json b/walker2d_medium_expert_sparse/diffusion_42/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0dbe23fd6a0f8f13f0bce60db5ef70d55bec9257 --- /dev/null +++ b/walker2d_medium_expert_sparse/diffusion_42/scheduler/scheduler_config.json @@ -0,0 +1,19 @@ +{ + "_class_name": "DDIMScheduler", + "_diffusers_version": "0.36.0", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "clip_sample": true, + "clip_sample_range": 1.0, + "dynamic_thresholding_ratio": 0.995, + "num_train_timesteps": 50000, + "prediction_type": "epsilon", + "rescale_betas_zero_snr": false, + "sample_max_value": 1.0, + "set_alpha_to_one": true, + "steps_offset": 0, + "thresholding": false, + "timestep_spacing": "leading", + "trained_betas": null +} diff --git a/walker2d_medium_expert_sparse/diffusion_42/used_config.yaml b/walker2d_medium_expert_sparse/diffusion_42/used_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b263b542d24a33c1f39a95bdf1ca5c22078cebe --- /dev/null +++ b/walker2d_medium_expert_sparse/diffusion_42/used_config.yaml @@ -0,0 +1,21 @@ +npz: /public/d4rl/sparse_datasets/diffusion_processed/walker2d_medium_expert_sparse_73_train.npz +out: /public/gormpo/models/walker2d_medium_expert_sparse_3/diffusion +epochs: 100 +batch: 512 +lr: 0.0002 +wd: 0.0 +timesteps: 50000 +checkpoint_every_epochs: 10 +seed: 0 +device: cuda +model: mlp +hidden: 512 +time_emb: 128 +layers: 3 +dropout: 0.0 +wandb: false +wandb_project: GORMPO +wandb_entity: jhhuang_chloe +wandb_run: unconditional_ddim_mlp_walker2d_medium_expert_sparse_73 +log_every: 100 +samples_every: 1 diff --git a/walker2d_medium_expert_sparse/diffusion_456/scheduler/scheduler_config.json b/walker2d_medium_expert_sparse/diffusion_456/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0dbe23fd6a0f8f13f0bce60db5ef70d55bec9257 --- /dev/null +++ b/walker2d_medium_expert_sparse/diffusion_456/scheduler/scheduler_config.json @@ -0,0 +1,19 @@ +{ + "_class_name": "DDIMScheduler", + "_diffusers_version": "0.36.0", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "clip_sample": true, + "clip_sample_range": 1.0, + "dynamic_thresholding_ratio": 0.995, + "num_train_timesteps": 50000, + "prediction_type": "epsilon", + "rescale_betas_zero_snr": false, + "sample_max_value": 1.0, + "set_alpha_to_one": true, + "steps_offset": 0, + "thresholding": false, + "timestep_spacing": "leading", + "trained_betas": null +} diff --git a/walker2d_medium_expert_sparse/diffusion_456/used_config.yaml b/walker2d_medium_expert_sparse/diffusion_456/used_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b263b542d24a33c1f39a95bdf1ca5c22078cebe --- /dev/null +++ b/walker2d_medium_expert_sparse/diffusion_456/used_config.yaml @@ -0,0 +1,21 @@ +npz: /public/d4rl/sparse_datasets/diffusion_processed/walker2d_medium_expert_sparse_73_train.npz +out: /public/gormpo/models/walker2d_medium_expert_sparse_3/diffusion +epochs: 100 +batch: 512 +lr: 0.0002 +wd: 0.0 +timesteps: 50000 +checkpoint_every_epochs: 10 +seed: 0 +device: cuda +model: mlp +hidden: 512 +time_emb: 128 +layers: 3 +dropout: 0.0 +wandb: false +wandb_project: GORMPO +wandb_entity: jhhuang_chloe +wandb_run: unconditional_ddim_mlp_walker2d_medium_expert_sparse_73 +log_every: 100 +samples_every: 1 diff --git a/walker2d_medium_expert_sparse/mbpo_based/diffusion/seed_42_0329_221335_walker2d_medium_expert_v2_gormpo_sparse_73/policy_walker2d-medium-expert-v2.pth b/walker2d_medium_expert_sparse/mbpo_based/diffusion/seed_42_0329_221335_walker2d_medium_expert_v2_gormpo_sparse_73/policy_walker2d-medium-expert-v2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mbpo_based/kde/seed_123_0329_131225_walker2d_medium_expert_v2_gormpo_sparse_73/policy_walker2d-medium-expert-v2.pth b/walker2d_medium_expert_sparse/mbpo_based/kde/seed_123_0329_131225_walker2d_medium_expert_v2_gormpo_sparse_73/policy_walker2d-medium-expert-v2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mbpo_based/kde/seed_456_0304_025033_walker2d_medium_expert_v2_gormpo_sparse_73/dynamics_model/model.pt b/walker2d_medium_expert_sparse/mbpo_based/kde/seed_456_0304_025033_walker2d_medium_expert_v2_gormpo_sparse_73/dynamics_model/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..962902698f193783fc6850c5eb99cb6702cdd5eb --- /dev/null +++ b/walker2d_medium_expert_sparse/mbpo_based/kde/seed_456_0304_025033_walker2d_medium_expert_v2_gormpo_sparse_73/dynamics_model/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e664534a90dd5b4942f69fa57699ed3146632c867e3c8d81566218ddbd3ff24 +size 3736742 diff --git a/walker2d_medium_expert_sparse/mbpo_based/kde/seed_456_0304_025033_walker2d_medium_expert_v2_gormpo_sparse_73/dynamics_model/normalizers.pt b/walker2d_medium_expert_sparse/mbpo_based/kde/seed_456_0304_025033_walker2d_medium_expert_v2_gormpo_sparse_73/dynamics_model/normalizers.pt new file mode 100644 index 0000000000000000000000000000000000000000..74d17645d19d16e6cbaa8d8c49b56e3f5c4698f4 --- /dev/null +++ b/walker2d_medium_expert_sparse/mbpo_based/kde/seed_456_0304_025033_walker2d_medium_expert_v2_gormpo_sparse_73/dynamics_model/normalizers.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31b48b50a591ca7ddb895c10bd2606a95e7c4fe50afe220fa6dbff0a5d3a818c +size 1584 diff --git a/walker2d_medium_expert_sparse/mbpo_based/neuralODE/seed_42_0329_042123_walker2d_medium_expert_v2_gormpo_sparse_73/policy_walker2d-medium-expert-v2.pth b/walker2d_medium_expert_sparse/mbpo_based/neuralODE/seed_42_0329_042123_walker2d_medium_expert_v2_gormpo_sparse_73/policy_walker2d-medium-expert-v2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mbpo_based/realnvp/seed_123_0330_000935_walker2d_medium_expert_v2_gormpo_sparse_73/policy_walker2d-medium-expert-v2.pth b/walker2d_medium_expert_sparse/mbpo_based/realnvp/seed_123_0330_000935_walker2d_medium_expert_v2_gormpo_sparse_73/policy_walker2d-medium-expert-v2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mbpo_based/vae/seed_123_0329_225119_walker2d_medium_expert_v2_gormpo_sparse_73/policy_walker2d-medium-expert-v2.pth b/walker2d_medium_expert_sparse/mbpo_based/vae/seed_123_0329_225119_walker2d_medium_expert_v2_gormpo_sparse_73/policy_walker2d-medium-expert-v2.pth new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_123×tamp_26-0318-110436/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_123×tamp_26-0318-110436/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..92fed46ef6eb6fb271624e89e9cab40e20447f7b --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_123×tamp_26-0318-110436/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 3.9479 +num rollout transitions: 50000, reward mean: 4.0274 +num rollout transitions: 50000, reward mean: 4.0732 +num rollout transitions: 50000, reward mean: 4.0489 +num rollout transitions: 50000, reward mean: 4.0241 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0172 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 4.0252 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 3.9832 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0272 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0404 +num rollout transitions: 50000, reward mean: 4.0200 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0260 +num rollout transitions: 50000, reward mean: 4.0338 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0329 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9837 +num rollout transitions: 50000, reward mean: 3.9796 +num rollout transitions: 50000, reward mean: 3.9682 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9888 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 4.0262 +---------------------------------------------------------------------------------- +| alpha | 0.257 | +| eval/episode_length | 149 | +| eval/episode_length_std | 22.9 | +| eval/normalized_episode_reward | 0.791 | +| eval/normalized_episode_reward_std | 0.685 | +| loss/actor | -263 | +| loss/alpha | -2.11 | +| loss/critic | 18.9 | +| penalty/lcb_mean | 0.954 | +| penalty/lcb_std | 1.22 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0216 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 4.0295 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0249 +num rollout transitions: 50000, reward mean: 4.0191 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0246 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0221 +num rollout transitions: 50000, reward mean: 4.0332 +num rollout transitions: 50000, reward mean: 4.0258 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0256 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0209 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0333 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0389 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9849 +num rollout transitions: 50000, reward mean: 4.0335 +num rollout transitions: 50000, reward mean: 4.0231 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0310 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0008 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 79.8 | +| eval/episode_length_std | 1.01 | +| eval/normalized_episode_reward | 3.7 | +| eval/normalized_episode_reward_std | -0.0039 | +| loss/actor | -412 | +| loss/alpha | 0.00372 | +| loss/critic | 29.3 | +| penalty/lcb_mean | 1.16 | +| penalty/lcb_std | 1.7 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0298 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 4.0254 +num rollout transitions: 50000, reward mean: 4.0253 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 4.0212 +num rollout transitions: 50000, reward mean: 4.0200 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 4.0185 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 4.0195 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0265 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0183 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0340 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0207 +num rollout transitions: 50000, reward mean: 4.0176 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0190 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0246 +num rollout transitions: 50000, reward mean: 4.0186 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0302 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0247 +num rollout transitions: 50000, reward mean: 4.0284 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0142 +---------------------------------------------------------------------------------- +| alpha | 0.144 | +| eval/episode_length | 370 | +| eval/episode_length_std | 123 | +| eval/normalized_episode_reward | 32.5 | +| eval/normalized_episode_reward_std | 12.3 | +| loss/actor | -410 | +| loss/alpha | 0.00572 | +| loss/critic | 30.6 | +| penalty/lcb_mean | 1.18 | +| penalty/lcb_std | 1.68 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0295 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0365 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0288 +num rollout transitions: 50000, reward mean: 4.0254 +num rollout transitions: 50000, reward mean: 4.0288 +num rollout transitions: 50000, reward mean: 4.0183 +num rollout transitions: 50000, reward mean: 4.0283 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 4.0172 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0344 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0227 +num rollout transitions: 50000, reward mean: 4.0298 +num rollout transitions: 50000, reward mean: 4.0253 +num rollout transitions: 50000, reward mean: 4.0211 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0224 +num rollout transitions: 50000, reward mean: 4.0364 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0212 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 4.0168 +num rollout transitions: 50000, reward mean: 4.0271 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0213 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0212 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 3.9949 +---------------------------------------------------------------------------------- +| alpha | 0.152 | +| eval/episode_length | 952 | +| eval/episode_length_std | 145 | +| eval/normalized_episode_reward | 90 | +| eval/normalized_episode_reward_std | 15.2 | +| loss/actor | -396 | +| loss/alpha | 0.00751 | +| loss/critic | 33.3 | +| penalty/lcb_mean | 1.24 | +| penalty/lcb_std | 1.71 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.02 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 4.0214 +num rollout transitions: 50000, reward mean: 4.0278 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0175 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0197 +num rollout transitions: 50000, reward mean: 4.0245 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0276 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0242 +num rollout transitions: 50000, reward mean: 4.0214 +num rollout transitions: 50000, reward mean: 4.0228 +num rollout transitions: 50000, reward mean: 4.0229 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 4.0214 +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 4.0242 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0206 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0279 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9921 +---------------------------------------------------------------------------------- +| alpha | 0.151 | +| eval/episode_length | 982 | +| eval/episode_length_std | 78.6 | +| eval/normalized_episode_reward | 94.6 | +| eval/normalized_episode_reward_std | 8.05 | +| loss/actor | -382 | +| loss/alpha | -0.0067 | +| loss/critic | 34.2 | +| penalty/lcb_mean | 1.23 | +| penalty/lcb_std | 1.74 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0202 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 4.0235 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 4.0278 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0341 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0189 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0200 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9842 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 3.9915 +---------------------------------------------------------------------------------- +| alpha | 0.147 | +| eval/episode_length | 453 | +| eval/episode_length_std | 397 | +| eval/normalized_episode_reward | 40.6 | +| eval/normalized_episode_reward_std | 40.6 | +| loss/actor | -373 | +| loss/alpha | -0.00846 | +| loss/critic | 34.8 | +| penalty/lcb_mean | 1.23 | +| penalty/lcb_std | 1.76 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 3.9856 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0248 +num rollout transitions: 50000, reward mean: 3.9846 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0186 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0319 +num rollout transitions: 50000, reward mean: 4.0256 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 3.9795 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0287 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0149 +num rollout transitions: 50000, reward mean: 4.0236 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0220 +num rollout transitions: 50000, reward mean: 4.0235 +num rollout transitions: 50000, reward mean: 4.0189 +num rollout transitions: 50000, reward mean: 4.0201 +num rollout transitions: 50000, reward mean: 4.0210 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0112 +---------------------------------------------------------------------------------- +| alpha | 0.144 | +| eval/episode_length | 733 | +| eval/episode_length_std | 290 | +| eval/normalized_episode_reward | 74 | +| eval/normalized_episode_reward_std | 31.7 | +| loss/actor | -368 | +| loss/alpha | 0.000596 | +| loss/critic | 35.4 | +| penalty/lcb_mean | 1.23 | +| penalty/lcb_std | 1.77 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0293 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0249 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0211 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9811 +num rollout transitions: 50000, reward mean: 4.0228 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0199 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0172 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0248 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 4.0207 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9788 +num rollout transitions: 50000, reward mean: 4.0280 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0172 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9878 +num rollout transitions: 50000, reward mean: 4.0155 +----------------------------------------------------------------------------------- +| alpha | 0.142 | +| eval/episode_length | 740 | +| eval/episode_length_std | 206 | +| eval/normalized_episode_reward | 73.7 | +| eval/normalized_episode_reward_std | 22.5 | +| loss/actor | -371 | +| loss/alpha | -6.09e-05 | +| loss/critic | 37 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.84 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 400000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 4.0218 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0259 +num rollout transitions: 50000, reward mean: 4.0292 +num rollout transitions: 50000, reward mean: 4.0217 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0317 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9765 +num rollout transitions: 50000, reward mean: 3.9888 +num rollout transitions: 50000, reward mean: 3.9845 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9702 +num rollout transitions: 50000, reward mean: 3.9804 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 3.9859 +num rollout transitions: 50000, reward mean: 3.9833 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 823 | +| eval/episode_length_std | 199 | +| eval/normalized_episode_reward | 83.9 | +| eval/normalized_episode_reward_std | 22 | +| loss/actor | -373 | +| loss/alpha | -0.00688 | +| loss/critic | 37.1 | +| penalty/lcb_mean | 1.25 | +| penalty/lcb_std | 1.87 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9888 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 3.9888 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0237 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9879 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 3.9817 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 4.0302 +num rollout transitions: 50000, reward mean: 4.0259 +num rollout transitions: 50000, reward mean: 4.0185 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0042 +---------------------------------------------------------------------------------- +| alpha | 0.137 | +| eval/episode_length | 990 | +| eval/episode_length_std | 53.3 | +| eval/normalized_episode_reward | 98 | +| eval/normalized_episode_reward_std | 6.73 | +| loss/actor | -373 | +| loss/alpha | 0.00663 | +| loss/critic | 37.7 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 1.9 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 3.9830 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0266 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0234 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 3.9837 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9853 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9811 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9860 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0196 +num rollout transitions: 50000, reward mean: 3.9905 +num rollout transitions: 50000, reward mean: 3.9826 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0258 +----------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 988 | +| eval/episode_length_std | 84.7 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 9.94 | +| loss/actor | -373 | +| loss/alpha | -0.000525 | +| loss/critic | 36.3 | +| penalty/lcb_mean | 1.22 | +| penalty/lcb_std | 1.89 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 550000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 3.9735 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9836 +num rollout transitions: 50000, reward mean: 3.9796 +num rollout transitions: 50000, reward mean: 3.9755 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0149 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0223 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0264 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 4.0234 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0266 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9819 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 3.9852 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0091 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 0.808 | +| loss/actor | -373 | +| loss/alpha | -0.00844 | +| loss/critic | 37.3 | +| penalty/lcb_mean | 1.24 | +| penalty/lcb_std | 1.89 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 3.9812 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9786 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0184 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9868 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0038 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 975 | +| eval/episode_length_std | 96 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 11.7 | +| loss/actor | -374 | +| loss/alpha | -0.0043 | +| loss/critic | 39 | +| penalty/lcb_mean | 1.28 | +| penalty/lcb_std | 1.96 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0382 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0199 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0226 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0197 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0233 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9993 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 697 | +| eval/episode_length_std | 382 | +| eval/normalized_episode_reward | 72.6 | +| eval/normalized_episode_reward_std | 44.1 | +| loss/actor | -375 | +| loss/alpha | 0.00409 | +| loss/critic | 38.6 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.95 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9794 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 3.9855 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 3.9791 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 4.0184 +num rollout transitions: 50000, reward mean: 4.0236 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0449 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9841 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9902 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 3.9778 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0010 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 950 | +| eval/episode_length_std | 140 | +| eval/normalized_episode_reward | 100 | +| eval/normalized_episode_reward_std | 16.1 | +| loss/actor | -376 | +| loss/alpha | 0.00108 | +| loss/critic | 39.7 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 1.97 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 3.9905 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 3.9860 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 4.0178 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0281 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0264 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9828 +num rollout transitions: 50000, reward mean: 4.0185 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 4.0226 +num rollout transitions: 50000, reward mean: 3.9854 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0275 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 4.0145 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 915 | +| eval/episode_length_std | 168 | +| eval/normalized_episode_reward | 93.5 | +| eval/normalized_episode_reward_std | 19.3 | +| loss/actor | -378 | +| loss/alpha | -0.00745 | +| loss/critic | 39 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.97 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9743 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0176 +num rollout transitions: 50000, reward mean: 3.9808 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9902 +num rollout transitions: 50000, reward mean: 3.9835 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9848 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9902 +num rollout transitions: 50000, reward mean: 4.0177 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0007 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 1.11 | +| loss/actor | -376 | +| loss/alpha | 0.00271 | +| loss/critic | 39.4 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 1.99 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9770 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 3.9762 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0291 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0283 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 3.9865 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 3.9851 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 3.9882 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0243 +num rollout transitions: 50000, reward mean: 3.9965 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.779 | +| loss/actor | -377 | +| loss/alpha | -0.012 | +| loss/critic | 36.8 | +| penalty/lcb_mean | 1.22 | +| penalty/lcb_std | 1.92 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 3.9848 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0183 +num rollout transitions: 50000, reward mean: 4.0210 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 4.0176 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0222 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0211 +num rollout transitions: 50000, reward mean: 4.0196 +num rollout transitions: 50000, reward mean: 4.0259 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 3.9757 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 4.0156 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0225 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0333 +num rollout transitions: 50000, reward mean: 4.0234 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 963 | +| eval/episode_length_std | 95.6 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 11.5 | +| loss/actor | -380 | +| loss/alpha | 0.00706 | +| loss/critic | 36 | +| penalty/lcb_mean | 1.21 | +| penalty/lcb_std | 1.91 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0196 +num rollout transitions: 50000, reward mean: 4.0211 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0202 +num rollout transitions: 50000, reward mean: 4.0249 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9796 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 3.9874 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0168 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0015 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 966 | +| eval/episode_length_std | 83.2 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 9.9 | +| loss/actor | -379 | +| loss/alpha | 0.0066 | +| loss/critic | 35.1 | +| penalty/lcb_mean | 1.21 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 3.9891 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9871 +num rollout transitions: 50000, reward mean: 3.9820 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0191 +num rollout transitions: 50000, reward mean: 3.9797 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0176 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9915 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0227 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 3.9838 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 4.0149 +num rollout transitions: 50000, reward mean: 4.0268 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0248 +num rollout transitions: 50000, reward mean: 3.9961 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.709 | +| loss/actor | -376 | +| loss/alpha | -0.0165 | +| loss/critic | 37.6 | +| penalty/lcb_mean | 1.25 | +| penalty/lcb_std | 1.98 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0149 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0175 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0191 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0213 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0280 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0261 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0252 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0049 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 975 | +| eval/episode_length_std | 82 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 10.5 | +| loss/actor | -375 | +| loss/alpha | -0.00488 | +| loss/critic | 36.4 | +| penalty/lcb_mean | 1.22 | +| penalty/lcb_std | 1.94 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9847 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0285 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 3.9902 +num rollout transitions: 50000, reward mean: 4.0246 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 4.0229 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0225 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0211 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 3.9945 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 998 | +| eval/episode_length_std | 18.4 | +| eval/normalized_episode_reward | 119 | +| eval/normalized_episode_reward_std | 2.47 | +| loss/actor | -374 | +| loss/alpha | 0.0104 | +| loss/critic | 37.8 | +| penalty/lcb_mean | 1.24 | +| penalty/lcb_std | 1.98 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9772 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0198 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0235 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0261 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0273 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0190 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0305 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 4.0103 +---------------------------------------------------------------------------------- +| alpha | 0.112 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.998 | +| loss/actor | -373 | +| loss/alpha | -0.00562 | +| loss/critic | 38.2 | +| penalty/lcb_mean | 1.23 | +| penalty/lcb_std | 2.01 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9770 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 3.9753 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 4.0229 +num rollout transitions: 50000, reward mean: 3.9850 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0330 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 3.9811 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0232 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 3.9708 +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0268 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0066 +----------------------------------------------------------------------------------- +| alpha | 0.111 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 120 | +| eval/normalized_episode_reward_std | 0.614 | +| loss/actor | -373 | +| loss/alpha | -0.000999 | +| loss/critic | 37.9 | +| penalty/lcb_mean | 1.24 | +| penalty/lcb_std | 1.98 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1250000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0211 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 3.9812 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0200 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0250 +num rollout transitions: 50000, reward mean: 4.0216 +num rollout transitions: 50000, reward mean: 4.0315 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0242 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0214 +num rollout transitions: 50000, reward mean: 3.9905 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9877 +num rollout transitions: 50000, reward mean: 3.9998 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.609 | +| loss/actor | -374 | +| loss/alpha | -0.00186 | +| loss/critic | 38 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 2 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9763 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9832 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 4.0064 +---------------------------------------------------------------------------------- +| alpha | 0.109 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.842 | +| loss/actor | -374 | +| loss/alpha | -0.00266 | +| loss/critic | 38.6 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 2.02 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0185 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 4.0302 +num rollout transitions: 50000, reward mean: 4.0276 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0073 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.345 | +| loss/actor | -371 | +| loss/alpha | 0.00473 | +| loss/critic | 40.5 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 2.08 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0217 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0229 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0239 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 3.9928 +---------------------------------------------------------------------------------- +| alpha | 0.115 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.633 | +| loss/actor | -370 | +| loss/alpha | 0.00376 | +| loss/critic | 43.7 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.14 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0231 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0298 +num rollout transitions: 50000, reward mean: 4.0377 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0168 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0201 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0051 +---------------------------------------------------------------------------------- +| alpha | 0.111 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.191 | +| loss/actor | -372 | +| loss/alpha | 0.00381 | +| loss/critic | 42 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.1 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 4.0218 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 4.0249 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0262 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0260 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0205 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0220 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.278 | +| loss/actor | -368 | +| loss/alpha | -0.0113 | +| loss/critic | 44.6 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.17 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0156 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 3.9790 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 3.9849 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0205 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0195 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0265 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 3.9829 +num rollout transitions: 50000, reward mean: 4.0191 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 4.0259 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0176 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9895 +---------------------------------------------------------------------------------- +| alpha | 0.111 | +| eval/episode_length | 981 | +| eval/episode_length_std | 88.6 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 10.2 | +| loss/actor | -370 | +| loss/alpha | -0.00295 | +| loss/critic | 41.2 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.08 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0299 +num rollout transitions: 50000, reward mean: 4.0206 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9878 +num rollout transitions: 50000, reward mean: 3.9920 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.963 | +| loss/actor | -368 | +| loss/alpha | 0.0102 | +| loss/critic | 46.1 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0216 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9808 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9915 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0168 +num rollout transitions: 50000, reward mean: 4.0266 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0276 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0156 +---------------------------------------------------------------------------------- +| alpha | 0.109 | +| eval/episode_length | 993 | +| eval/episode_length_std | 38 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 4.43 | +| loss/actor | -369 | +| loss/alpha | -0.00427 | +| loss/critic | 44.3 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.19 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 3.9856 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 4.0199 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0193 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0271 +num rollout transitions: 50000, reward mean: 4.0248 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9811 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0195 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0226 +num rollout transitions: 50000, reward mean: 4.0196 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0195 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 4.0177 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0149 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0246 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 4.0202 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0320 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0283 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0016 +---------------------------------------------------------------------------------- +| alpha | 0.112 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.379 | +| loss/actor | -373 | +| loss/alpha | 0.00725 | +| loss/critic | 45.6 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0230 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 4.0229 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0200 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0247 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 3.9858 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0215 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 3.9838 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 4.0255 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0048 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.146 | +| loss/actor | -371 | +| loss/alpha | -0.00559 | +| loss/critic | 45.5 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0210 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0288 +num rollout transitions: 50000, reward mean: 4.0290 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0270 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9830 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 4.0231 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0150 +----------------------------------------------------------------------------------- +| alpha | 0.113 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 1.77 | +| loss/actor | -365 | +| loss/alpha | -0.000416 | +| loss/critic | 44.8 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.17 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1850000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0197 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 3.9827 +num rollout transitions: 50000, reward mean: 3.9858 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0315 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0190 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0298 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0291 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0151 +---------------------------------------------------------------------------------- +| alpha | 0.112 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.516 | +| loss/actor | -366 | +| loss/alpha | 0.0103 | +| loss/critic | 44.7 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.18 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9826 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9869 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 3.9915 +num rollout transitions: 50000, reward mean: 4.0233 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 4.0175 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0209 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0149 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0232 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9969 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 1.06 | +| loss/actor | -367 | +| loss/alpha | 0.00261 | +| loss/critic | 46.3 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9860 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0212 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 4.0211 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0219 +num rollout transitions: 50000, reward mean: 4.0202 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0318 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0325 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0205 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0245 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0248 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9963 +---------------------------------------------------------------------------------- +| alpha | 0.118 | +| eval/episode_length | 996 | +| eval/episode_length_std | 44.8 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 5.5 | +| loss/actor | -369 | +| loss/alpha | -0.00624 | +| loss/critic | 46.5 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0236 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 4.0268 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0284 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9838 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 4.0103 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 967 | +| eval/episode_length_std | 103 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 12.2 | +| loss/actor | -367 | +| loss/alpha | 0.000919 | +| loss/critic | 49.1 | +| penalty/lcb_mean | 1.45 | +| penalty/lcb_std | 2.26 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0197 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0232 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0036 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.134 | +| loss/actor | -368 | +| loss/alpha | 0.00375 | +| loss/critic | 47.4 | +| penalty/lcb_mean | 1.43 | +| penalty/lcb_std | 2.25 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9878 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0193 +num rollout transitions: 50000, reward mean: 3.9846 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0168 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 3.9846 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0270 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0202 +num rollout transitions: 50000, reward mean: 4.0223 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 3.9871 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9908 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.314 | +| loss/actor | -369 | +| loss/alpha | -0.00675 | +| loss/critic | 46.1 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0224 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0149 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0168 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0251 +num rollout transitions: 50000, reward mean: 4.0220 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 4.0233 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0221 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0222 +---------------------------------------------------------------------------------- +| alpha | 0.113 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.353 | +| loss/actor | -368 | +| loss/alpha | -0.00206 | +| loss/critic | 47.2 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9853 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0194 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 3.9851 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0186 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 4.0205 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 3.9906 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 994 | +| eval/episode_length_std | 36.6 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 4.37 | +| loss/actor | -368 | +| loss/alpha | 0.00823 | +| loss/critic | 51.2 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0216 +num rollout transitions: 50000, reward mean: 4.0237 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 4.0205 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 3.9860 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0235 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 3.9868 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0177 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9941 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.287 | +| loss/actor | -365 | +| loss/alpha | 0.00586 | +| loss/critic | 47.9 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.23 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0237 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9820 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9883 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0236 +num rollout transitions: 50000, reward mean: 4.0226 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0271 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0188 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0037 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 991 | +| eval/episode_length_std | 60.6 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 7.26 | +| loss/actor | -364 | +| loss/alpha | -0.00789 | +| loss/critic | 49.7 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 3.9832 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0215 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 4.0196 +num rollout transitions: 50000, reward mean: 4.0195 +num rollout transitions: 50000, reward mean: 4.0257 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9882 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0169 +---------------------------------------------------------------------------------- +| alpha | 0.115 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.282 | +| loss/actor | -366 | +| loss/alpha | 0.0107 | +| loss/critic | 50 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.3 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 3.9878 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0178 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9908 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.458 | +| loss/actor | -365 | +| loss/alpha | -0.0123 | +| loss/critic | 50.1 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0283 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9857 +num rollout transitions: 50000, reward mean: 3.9847 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0178 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0212 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0198 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9973 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 979 | +| eval/episode_length_std | 86.2 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 10.8 | +| loss/actor | -369 | +| loss/alpha | 0.00341 | +| loss/critic | 49.8 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9826 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0195 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0198 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9844 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 4.0178 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 4.0223 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0101 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.449 | +| loss/actor | -370 | +| loss/alpha | -0.00447 | +| loss/critic | 48.3 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0199 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 3.9795 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 3.9853 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 4.0059 +---------------------------------------------------------------------------------- +| alpha | 0.11 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.311 | +| loss/actor | -371 | +| loss/alpha | -0.00533 | +| loss/critic | 49.9 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 4.0208 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0193 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0200 +num rollout transitions: 50000, reward mean: 4.0231 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0214 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0207 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 3.9938 +---------------------------------------------------------------------------------- +| alpha | 0.109 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.507 | +| loss/actor | -372 | +| loss/alpha | 0.00922 | +| loss/critic | 48 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.27 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0172 +num rollout transitions: 50000, reward mean: 4.0200 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9831 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 3.9850 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0149 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9849 +num rollout transitions: 50000, reward mean: 3.9867 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 3.9867 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 3.9996 +---------------------------------------------------------------------------------- +| alpha | 0.104 | +| eval/episode_length | 999 | +| eval/episode_length_std | 7.06 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 1.26 | +| loss/actor | -373 | +| loss/alpha | -0.00666 | +| loss/critic | 48 | +| penalty/lcb_mean | 1.45 | +| penalty/lcb_std | 2.3 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0183 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 3.9857 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9799 +num rollout transitions: 50000, reward mean: 3.9858 +num rollout transitions: 50000, reward mean: 3.9924 +---------------------------------------------------------------------------------- +| alpha | 0.105 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.738 | +| loss/actor | -370 | +| loss/alpha | -0.0129 | +| loss/critic | 49 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9860 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9817 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9862 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9877 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0018 +---------------------------------------------------------------------------------- +| alpha | 0.108 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.712 | +| loss/actor | -367 | +| loss/alpha | 0.0265 | +| loss/critic | 51.5 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 3.9869 +num rollout transitions: 50000, reward mean: 3.9869 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9971 +---------------------------------------------------------------------------------- +| alpha | 0.106 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.385 | +| loss/actor | -366 | +| loss/alpha | -0.0303 | +| loss/critic | 50.6 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.37 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0226 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0189 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 3.9902 +num rollout transitions: 50000, reward mean: 3.9878 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 4.0204 +---------------------------------------------------------------------------------- +| alpha | 0.101 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.4 | +| loss/actor | -368 | +| loss/alpha | 0.0123 | +| loss/critic | 50 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0275 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0168 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0004 +---------------------------------------------------------------------------------- +| alpha | 0.102 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.354 | +| loss/actor | -370 | +| loss/alpha | 0.0194 | +| loss/critic | 50.4 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0019 +---------------------------------------------------------------------------------- +| alpha | 0.102 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.404 | +| loss/actor | -363 | +| loss/alpha | 0.00297 | +| loss/critic | 51.9 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 128171.04s diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_123×tamp_26-0318-110436/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_123×tamp_26-0318-110436/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_123×tamp_26-0318-110436/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_123×tamp_26-0318-110436/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..40f49573387e82a9625bd9e7442e73e989b2d442 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_123×tamp_26-0318-110436/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_diffusion.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 123, "device": "cuda:7", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_123×tamp_26-0112-030558/model", "reward_penalty_coef": 0.5, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/diffusion/checkpoint.pt", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "density_model": "diffusion", "data_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "penalty_type": "tanh", "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "diffusion"} diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_123×tamp_26-0318-110436/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_123×tamp_26-0318-110436/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..770ecdb76956ce1b9f99ca945165a2fc66f91d45 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_123×tamp_26-0318-110436/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.2569602797505301,149.16,22.917556588781448,0.7910023558757269,0.6846615316440469,-262.91484590656927,-2.1099551502804035,18.941839337577377,0.9539310742166647,1.2174721044818249,1.0,0.0,50000.0,4.008804721709407,50000 +0.13086671972021433,79.8,1.0099504938362078,3.7013666961408944,-0.0039029976164907405,-412.340385280759,0.003715914236418706,29.346568571071735,1.1555151186180315,1.702638341746326,1.0,0.0,50000.0,4.01434281589719,100000 +0.14397376851141422,369.83,122.81580150778645,32.522625751923194,12.279642325459518,-410.1812615545643,0.005722395344860824,30.625820516204815,1.1795785833608923,1.6824834882783846,1.0,0.0,50000.0,4.013266513939563,150000 +0.15214940165013013,951.99,145.26716731594928,90.02993586814809,15.178741053711118,-396.19484931274667,0.007506143559515514,33.31022395774852,1.23935077428101,1.7083978927528862,1.0,0.0,50000.0,4.016496500726141,200000 +0.15118354624092506,982.5,78.58504946871255,94.59544687232697,8.051976394106083,-382.00492953491255,-0.00670051597315801,34.196661952285,1.2255371724677182,1.7383604956686691,1.0,0.0,50000.0,4.012703073621762,250000 +0.1471114114227906,452.57,397.1588915031364,40.56289542506497,40.60089748221953,-372.88679790221926,-0.00846240766935055,34.772329122944534,1.2261212625849414,1.756086511439084,1.0,0.0,50000.0,4.008283459717478,300000 +0.14362991789460425,733.47,290.1108565703807,74.02950956438943,31.674089902447488,-367.9412748223834,0.0005961331483721913,35.40371496828058,1.2271078076303004,1.773877491171364,1.0,0.0,50000.0,4.006803972411574,350000 +0.1417946846204983,739.94,206.45051804245975,73.7041246068258,22.464532091690316,-371.1777701910424,-6.093910034738867e-05,36.9708730561638,1.2662532348990598,1.8392467483306,1.0,0.0,50000.0,4.006791781988166,400000 +0.1409734567365026,822.85,199.38351862679124,83.93590627275536,22.01441945790402,-372.96174581970155,-0.006877446343153664,37.1156261184496,1.250947266532182,1.866195277452466,1.0,0.0,50000.0,4.002781707602156,450000 +0.13721801004320794,989.78,53.342774581005806,97.95911565779944,6.732078891797581,-373.1268988903833,0.006633425579927985,37.709595372066566,1.257499902604815,1.8983266144847872,1.0,0.0,50000.0,4.002503584031367,500000 +0.13151945477410965,987.91,84.66216333167964,105.06730437765557,9.939865127565717,-372.59107279663,-0.0005253386221826968,36.30548227644027,1.2163241503691713,1.8879108668207976,1.0,0.0,50000.0,4.001925383358205,550000 +0.12879594728037538,1000.0,0.0,104.35090267379805,0.8083402073369654,-372.8965132238744,-0.008439444953017036,37.29304559206014,1.240310036520972,1.8946603827965098,1.0,0.0,50000.0,4.00061840032047,600000 +0.1272888962183914,974.81,95.99205123342246,107.47579539458584,11.729344985317608,-374.13414314880697,-0.004297469076700359,38.9525898677252,1.280204203420893,1.9592352003705524,1.0,0.0,50000.0,4.001286720610441,650000 +0.12653302696958077,697.38,382.41720097296877,72.63251900061223,44.09737759998796,-375.17153708557106,0.004088032592944912,38.64077164022458,1.2730661470806737,1.945636404213922,1.0,0.0,50000.0,4.003599379098962,700000 +0.12767828645467696,949.59,139.5947774811078,100.32450349088815,16.142104537719334,-375.7560533972164,0.0010770870185271533,39.71782128295974,1.2936603452551083,1.9725133888769295,1.0,0.0,50000.0,4.001773199673773,750000 +0.1242950022324916,915.29,167.5127036973614,93.49270187216591,19.287837566043564,-377.82792092040955,-0.007445704812407321,39.023276597862484,1.2692260796522967,1.9717889479589608,1.0,0.0,50000.0,4.004554706013524,800000 +0.12426104018762793,1000.0,0.0,107.95027306505919,1.106077974684868,-376.0800710394335,0.002714476915933178,39.38901392181403,1.2894052938079665,1.9863429577899074,1.0,0.0,50000.0,4.001351974481057,850000 +0.11744937938362443,1000.0,0.0,110.81922451794448,0.7791642693286251,-377.45201113891517,-0.012047551082633291,36.78289013780632,1.2238512346088752,1.9163424479126925,1.0,0.0,50000.0,4.003940844534206,900000 +0.11586433856189267,962.94,95.57550104498539,103.88947077579725,11.511719910308829,-379.8844577197249,0.007055957059189627,36.04443597154601,1.213774112929095,1.9127446720731074,1.0,0.0,50000.0,4.007550694217074,950000 +0.11603019417449698,965.66,83.16636579771898,108.92077259135834,9.89682301322767,-379.29706278076327,0.006602847759090514,35.09398337396583,1.2130828578019188,1.8833162376844976,1.0,0.0,50000.0,4.005868452714623,1000000 +0.1190363067439201,1000.0,0.0,110.2288270958429,0.7088525113924105,-375.8529324163824,-0.016528314115442325,37.6194084413337,1.2450656741345025,1.9788227754795489,1.0,0.0,50000.0,4.0048627946999185,1050000 +0.11399095051348218,975.41,82.0298841886297,112.53380732467973,10.542338380125859,-375.48937288451714,-0.004883535950593696,36.36132871694568,1.2235149646699255,1.9418348423361746,1.0,0.0,50000.0,4.00797955724357,1100000 +0.11580512761280087,998.15,18.40726758647247,118.534226036468,2.47479410369682,-374.0310609814488,0.010416776951700143,37.78812670293804,1.2375861376714565,1.981176465879711,1.0,0.0,50000.0,4.00399160367614,1150000 +0.11154421611964632,1000.0,0.0,115.207558028444,0.9980071214009208,-373.37662739258,-0.005615338999889659,38.230901662407014,1.2289266223144604,2.007225360633128,1.0,0.0,50000.0,4.006445178359102,1200000 +0.11076298432081864,1000.0,0.0,119.95846899334981,0.6135414989527358,-372.6675862335182,-0.0009991501854359468,37.93344444526673,1.2406599675249865,1.9785697744357467,1.0,0.0,50000.0,4.002467725518476,1250000 +0.11446648420363514,1000.0,0.0,114.55416299398237,0.6085441723782615,-373.72503251892397,-0.001857529690116677,38.01955102279746,1.2649256220197647,1.9968203834533684,1.0,0.0,50000.0,4.006213509768509,1300000 +0.10927211612000952,1000.0,0.0,111.12822317817594,0.8420812104197555,-373.80241772033855,-0.0026573028564824885,38.55043833541841,1.2642836018407277,2.0174193175184785,1.0,0.0,50000.0,4.000970668452011,1350000 +0.11433253828033917,1000.0,0.0,109.40954137157641,0.34499000874392277,-371.3033276416072,0.004732270140237969,40.52622574514357,1.2974389438557508,2.0760805757594127,1.0,0.0,50000.0,4.005214745820703,1400000 +0.114988910821676,1000.0,0.0,114.8112064824716,0.6334379039489506,-370.0399181524662,0.0037634728788956155,43.66169459741514,1.3654623713278706,2.139950856317322,1.0,0.0,50000.0,4.005501908673204,1450000 +0.11093232823982772,1000.0,0.0,112.46187611456008,0.1905563188888164,-372.2705479028304,0.0038137106660379866,42.02618128549596,1.3184582690012583,2.0983237293195236,1.0,0.0,50000.0,4.005882104567779,1500000 +0.1140657761630412,1000.0,0.0,110.15703097078429,0.2782506508256359,-368.32177158935815,-0.011344712724741517,44.58848374465875,1.3546598614680847,2.166358288680286,1.0,0.0,50000.0,4.007829473473741,1550000 +0.11058555632933817,981.44,88.6001489840734,109.28354611171245,10.200130906057067,-370.160459737544,-0.0029531461853162825,41.18703958246232,1.321743825953009,2.0790284080755526,1.0,0.0,50000.0,4.004601059367774,1600000 +0.11581130776330678,1000.0,0.0,108.62170823627592,0.9633168502562222,-367.7789735388154,0.0101886847422643,46.094036282539335,1.3803576803350457,2.2132295905304202,1.0,0.0,50000.0,4.0049129257188225,1650000 +0.10921223311677597,993.46,38.02115726802645,108.93559273994504,4.432523384583608,-369.36739401428804,-0.004265388914681924,44.29295114648782,1.3586679069340202,2.188180020891427,1.0,0.0,50000.0,4.003961919297266,1700000 +0.11232447602838348,1000.0,0.0,111.65804291807304,0.3790876487368669,-372.50234821533564,0.007250342838727012,45.603454873562775,1.3736897130560768,2.214715817914018,1.0,0.0,50000.0,4.008760028513074,1750000 +0.1137376044340422,1000.0,0.0,111.9382859611421,0.14642083207451598,-371.45663008667026,-0.005585000726990472,45.5456559873586,1.3826356243693616,2.2066585294389323,1.0,0.0,50000.0,4.0052802696400285,1800000 +0.11343816916942491,1000.0,0.0,111.80807260535599,1.7692550724464633,-364.7453685162365,-0.00041634349487715576,44.77242828479759,1.3697805872678686,2.166898779982329,1.0,0.0,50000.0,4.005891300640415,1850000 +0.11157268400505066,1000.0,0.0,112.55650657418397,0.5157842957083071,-365.74836434814506,0.010262302083484864,44.715121528548785,1.3783262950682686,2.177175699149389,1.0,0.0,50000.0,4.005450199027857,1900000 +0.11620978216052015,1000.0,0.0,108.90110615863952,1.0551206764076395,-366.9647533605975,0.0026076995486020048,46.32057946241389,1.3931620593953036,2.2115064288151087,1.0,0.0,50000.0,4.005320708661638,1950000 +0.11843075131699417,995.5,44.7744346697979,112.53695908611677,5.4960985264910995,-368.88263679626397,-0.006236873506009463,46.538023590354676,1.4135068566024422,2.2202494056534126,1.0,0.0,50000.0,4.007137593364525,2000000 +0.11932801265329174,966.54,103.17649150848267,108.29165858056405,12.225842947856027,-366.92803861633075,0.0009191950928419552,49.08142734029704,1.4549946274137517,2.258543640278628,1.0,0.0,50000.0,4.007752271983789,2050000 +0.11936389200076561,1000.0,0.0,114.41562117545739,0.13449040571640933,-367.8395086212136,0.003751206403523788,47.431591363029334,1.4265368631017257,2.248370658047285,1.0,0.0,50000.0,4.003543368690789,2100000 +0.11376726318448659,1000.0,0.0,113.81736192132408,0.31355809276583313,-368.88677593627864,-0.006747145491987498,46.06366186187714,1.397251119536181,2.2160876664888756,1.0,0.0,50000.0,4.003176848397971,2150000 +0.11348550005480655,1000.0,0.0,113.55099337284116,0.35344631278030725,-367.6902165557866,-0.002064229842871561,47.21754706775642,1.4131838185119563,2.239740693289023,1.0,0.0,50000.0,4.009487164981165,2200000 +0.11986632187038727,993.65,36.61840384287661,110.79845813559275,4.372748458927642,-367.96767946228243,0.008225241112783895,51.157294102039344,1.490774699430476,2.3370945164561077,1.0,0.0,50000.0,4.003569114893272,2250000 +0.11950121334284365,1000.0,0.0,110.74895664405753,0.2867482102649713,-364.6802585900891,0.005859840607568659,47.90267981861136,1.4649509023714335,2.226392384002207,1.0,0.0,50000.0,4.005178852771915,2300000 +0.11928659750848922,991.39,60.61252263352846,111.11556888583942,7.260646097950631,-363.71430534118383,-0.007886751927807994,49.71494141286919,1.4927503229331907,2.2766142259049436,1.0,0.0,50000.0,4.0055814800693765,2350000 +0.11516787696719223,1000.0,0.0,110.16480184546342,0.2822451509456943,-366.23467188414753,0.01072631173990679,50.04115358985937,1.4898371410536935,2.301340639486257,1.0,0.0,50000.0,4.005624905319749,2400000 +0.11353003003657049,1000.0,0.0,114.69738686736117,0.4580691936183308,-365.27944359924487,-0.01231841233061629,50.137522872028974,1.4961874201369312,2.321348136091202,1.0,0.0,50000.0,4.001329256758822,2450000 +0.1158003960299481,978.75,86.24504333583467,111.77434593599807,10.843667780603013,-369.0021782214368,0.003411206333600061,49.848277942410064,1.4986352739154993,2.3167634167027344,1.0,0.0,50000.0,4.003924411594296,2500000 +0.11421341140464035,1000.0,0.0,115.41482580279889,0.4494979469781802,-370.48511335082105,-0.0044725479546564055,48.3493453925318,1.4711192551231378,2.2802755272769915,1.0,0.0,50000.0,4.0048893663572676,2550000 +0.10992564061999295,1000.0,0.0,115.9144127166889,0.31081774598652695,-371.1665436669931,-0.005334057018049185,49.85901994449627,1.4854409599089595,2.332338754572802,1.0,0.0,50000.0,4.003930595694816,2600000 +0.10911109111070591,1000.0,0.0,115.08029110863814,0.5074900254536597,-371.64099673583877,0.009224903085231745,48.01486591650008,1.4783332749378566,2.2742510154724194,1.0,0.0,50000.0,4.006610729692851,2650000 +0.1043675620405368,999.29,7.064410803457003,116.18404317563974,1.25691269690946,-373.40690248779066,-0.006656091121696051,48.00264514352762,1.4547089196217236,2.29816304010152,1.0,0.0,50000.0,4.001725945135117,2700000 +0.1047217597588896,1000.0,0.0,114.69458184028376,0.7379180143312473,-370.36865112181425,-0.01287818292949375,48.98084204151194,1.4782155264377583,2.3064754952991033,1.0,0.0,50000.0,4.002975265067101,2750000 +0.10810873761549608,1000.0,0.0,113.37880725007108,0.7121943542726056,-366.8118822595242,0.026502747114337966,51.47439349647554,1.528731584483383,2.3833886908018234,1.0,0.0,50000.0,3.998618185621501,2800000 +0.10645356605455289,1000.0,0.0,114.44795381178983,0.3854037318141732,-365.79070693175896,-0.030315264056846748,50.64714564941347,1.4865043902480357,2.3667024272740544,1.0,0.0,50000.0,4.00343044574418,2850000 +0.10142718180492435,1000.0,0.0,116.51007045942117,0.4004127758158774,-368.3467670440732,0.012264961577132062,50.036580651396996,1.480592982825033,2.347790437288291,1.0,0.0,50000.0,4.003982807804823,2900000 +0.10161906777560699,1000.0,0.0,114.90441790594994,0.35368827900653566,-369.7939878716994,0.019399811185561104,50.43047655942923,1.5134935234117466,2.347788296520686,1.0,0.0,50000.0,4.0058774809123046,2950000 +0.10223200381487672,1000.0,0.0,115.48610369226328,0.4037671500095815,-362.7499290130596,0.0029747274115687946,51.934140431098825,1.530951429852261,2.382976044757365,1.0,0.0,50000.0,4.00312183810687,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_42×tamp_26-0317-010307/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_42×tamp_26-0317-010307/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6931c9bf695e835a821794f428f888b3e31df89 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_42×tamp_26-0317-010307/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 3.9244 +num rollout transitions: 50000, reward mean: 4.0252 +num rollout transitions: 50000, reward mean: 4.0740 +num rollout transitions: 50000, reward mean: 4.0530 +num rollout transitions: 50000, reward mean: 4.0537 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0186 +num rollout transitions: 50000, reward mean: 4.0293 +num rollout transitions: 50000, reward mean: 4.0353 +num rollout transitions: 50000, reward mean: 4.0457 +num rollout transitions: 50000, reward mean: 4.0307 +num rollout transitions: 50000, reward mean: 4.0464 +num rollout transitions: 50000, reward mean: 4.0225 +num rollout transitions: 50000, reward mean: 4.0455 +num rollout transitions: 50000, reward mean: 4.0349 +num rollout transitions: 50000, reward mean: 4.0293 +num rollout transitions: 50000, reward mean: 4.0237 +num rollout transitions: 50000, reward mean: 4.0216 +num rollout transitions: 50000, reward mean: 4.0197 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0316 +num rollout transitions: 50000, reward mean: 4.0360 +num rollout transitions: 50000, reward mean: 4.0435 +num rollout transitions: 50000, reward mean: 4.0567 +num rollout transitions: 50000, reward mean: 4.0367 +num rollout transitions: 50000, reward mean: 4.0338 +num rollout transitions: 50000, reward mean: 4.0475 +num rollout transitions: 50000, reward mean: 4.0412 +num rollout transitions: 50000, reward mean: 4.0319 +num rollout transitions: 50000, reward mean: 4.0364 +num rollout transitions: 50000, reward mean: 4.0427 +num rollout transitions: 50000, reward mean: 4.0304 +num rollout transitions: 50000, reward mean: 4.0194 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 3.9804 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9940 +---------------------------------------------------------------------------------- +| alpha | 0.26 | +| eval/episode_length | 131 | +| eval/episode_length_std | 24.1 | +| eval/normalized_episode_reward | 1.91 | +| eval/normalized_episode_reward_std | 0.778 | +| loss/actor | -256 | +| loss/alpha | -1.9 | +| loss/critic | 18.6 | +| penalty/lcb_mean | 0.961 | +| penalty/lcb_std | 1.17 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.02 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 3.9849 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9869 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 3.9608 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 4.0185 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9829 +num rollout transitions: 50000, reward mean: 3.9865 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0229 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 3.9708 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0199 +num rollout transitions: 50000, reward mean: 4.0261 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9851 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0219 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0010 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 233 | +| eval/episode_length_std | 114 | +| eval/normalized_episode_reward | 13 | +| eval/normalized_episode_reward_std | 11.5 | +| loss/actor | -380 | +| loss/alpha | 0.0213 | +| loss/critic | 30.4 | +| penalty/lcb_mean | 1.2 | +| penalty/lcb_std | 1.64 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 4.0301 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0273 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0272 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 4.0195 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0260 +num rollout transitions: 50000, reward mean: 4.0348 +num rollout transitions: 50000, reward mean: 4.0219 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0295 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 3.9883 +num rollout transitions: 50000, reward mean: 4.0248 +num rollout transitions: 50000, reward mean: 4.0258 +num rollout transitions: 50000, reward mean: 4.0236 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0252 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0234 +num rollout transitions: 50000, reward mean: 4.0362 +num rollout transitions: 50000, reward mean: 4.0256 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 4.0235 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0242 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 182 | +| eval/episode_length_std | 67.7 | +| eval/normalized_episode_reward | 3 | +| eval/normalized_episode_reward_std | 4.3 | +| loss/actor | -393 | +| loss/alpha | -0.00695 | +| loss/critic | 28.9 | +| penalty/lcb_mean | 1.18 | +| penalty/lcb_std | 1.59 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0321 +num rollout transitions: 50000, reward mean: 4.0322 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0225 +num rollout transitions: 50000, reward mean: 4.0266 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0361 +num rollout transitions: 50000, reward mean: 4.0190 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 4.0235 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 4.0271 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 3.9857 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0201 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0257 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0336 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9841 +num rollout transitions: 50000, reward mean: 4.0225 +num rollout transitions: 50000, reward mean: 3.9852 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 4.0193 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0188 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0270 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0222 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0058 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 198 | +| eval/episode_length_std | 80.7 | +| eval/normalized_episode_reward | 11.8 | +| eval/normalized_episode_reward_std | 9.16 | +| loss/actor | -392 | +| loss/alpha | 0.0136 | +| loss/critic | 31.1 | +| penalty/lcb_mean | 1.22 | +| penalty/lcb_std | 1.65 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0238 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0281 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0333 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 4.0224 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0434 +num rollout transitions: 50000, reward mean: 4.0218 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0303 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0227 +num rollout transitions: 50000, reward mean: 4.0249 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0272 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0232 +num rollout transitions: 50000, reward mean: 4.0323 +num rollout transitions: 50000, reward mean: 4.0184 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 4.0218 +num rollout transitions: 50000, reward mean: 4.0258 +num rollout transitions: 50000, reward mean: 4.0346 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0194 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0252 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0237 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0253 +---------------------------------------------------------------------------------- +| alpha | 0.146 | +| eval/episode_length | 274 | +| eval/episode_length_std | 227 | +| eval/normalized_episode_reward | 21.2 | +| eval/normalized_episode_reward_std | 23.7 | +| loss/actor | -373 | +| loss/alpha | 0.0139 | +| loss/critic | 34.2 | +| penalty/lcb_mean | 1.25 | +| penalty/lcb_std | 1.73 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.02 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0241 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0258 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0278 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 4.0266 +num rollout transitions: 50000, reward mean: 4.0189 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0276 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0367 +num rollout transitions: 50000, reward mean: 4.0279 +num rollout transitions: 50000, reward mean: 4.0358 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0242 +num rollout transitions: 50000, reward mean: 4.0198 +num rollout transitions: 50000, reward mean: 4.0306 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0218 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0205 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0177 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 4.0239 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 4.0304 +num rollout transitions: 50000, reward mean: 4.0305 +num rollout transitions: 50000, reward mean: 4.0238 +num rollout transitions: 50000, reward mean: 4.0230 +num rollout transitions: 50000, reward mean: 4.0013 +---------------------------------------------------------------------------------- +| alpha | 0.147 | +| eval/episode_length | 378 | +| eval/episode_length_std | 337 | +| eval/normalized_episode_reward | 32.6 | +| eval/normalized_episode_reward_std | 35.5 | +| loss/actor | -359 | +| loss/alpha | -0.00775 | +| loss/critic | 33.2 | +| penalty/lcb_mean | 1.25 | +| penalty/lcb_std | 1.69 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.02 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0177 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0207 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0249 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0270 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0270 +num rollout transitions: 50000, reward mean: 4.0207 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0130 +---------------------------------------------------------------------------------- +| alpha | 0.149 | +| eval/episode_length | 135 | +| eval/episode_length_std | 19.1 | +| eval/normalized_episode_reward | 7.43 | +| eval/normalized_episode_reward_std | 1.63 | +| loss/actor | -351 | +| loss/alpha | 0.0128 | +| loss/critic | 34.6 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.72 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 4.0263 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0186 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0191 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0212 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0276 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9782 +num rollout transitions: 50000, reward mean: 4.0043 +---------------------------------------------------------------------------------- +| alpha | 0.151 | +| eval/episode_length | 996 | +| eval/episode_length_std | 42.1 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 5.01 | +| loss/actor | -343 | +| loss/alpha | -0.0138 | +| loss/critic | 37.3 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 1.82 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9888 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 4.0222 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 4.0184 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 3.9869 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0064 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 989 | +| eval/episode_length_std | 60.5 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 6.98 | +| loss/actor | -341 | +| loss/alpha | 0.000547 | +| loss/critic | 39.1 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 1.93 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0243 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 3.9803 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 3.9844 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0177 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0220 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0222 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 3.9833 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0328 +num rollout transitions: 50000, reward mean: 4.0511 +num rollout transitions: 50000, reward mean: 4.0254 +num rollout transitions: 50000, reward mean: 4.0226 +----------------------------------------------------------------------------------- +| alpha | 0.148 | +| eval/episode_length | 724 | +| eval/episode_length_std | 258 | +| eval/normalized_episode_reward | 73.4 | +| eval/normalized_episode_reward_std | 29 | +| loss/actor | -340 | +| loss/alpha | -0.000159 | +| loss/critic | 40.5 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 1.93 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 500000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0224 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 4.0200 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0281 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 4.0303 +num rollout transitions: 50000, reward mean: 4.0208 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 3.9982 +---------------------------------------------------------------------------------- +| alpha | 0.144 | +| eval/episode_length | 965 | +| eval/episode_length_std | 103 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 12.7 | +| loss/actor | -344 | +| loss/alpha | -0.00448 | +| loss/critic | 37.4 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 1.86 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9819 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0254 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9795 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 3.9864 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 4.0276 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 3.9882 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9806 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9852 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0104 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 991 | +| eval/episode_length_std | 53.1 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 6.9 | +| loss/actor | -347 | +| loss/alpha | -0.00177 | +| loss/critic | 40 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 1.96 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0212 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 4.0257 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 3.9805 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0214 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9815 +num rollout transitions: 50000, reward mean: 3.9841 +num rollout transitions: 50000, reward mean: 3.9741 +num rollout transitions: 50000, reward mean: 3.9784 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9965 +---------------------------------------------------------------------------------- +| alpha | 0.142 | +| eval/episode_length | 474 | +| eval/episode_length_std | 403 | +| eval/normalized_episode_reward | 46.9 | +| eval/normalized_episode_reward_std | 46.9 | +| loss/actor | -342 | +| loss/alpha | 0.00807 | +| loss/critic | 41.8 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 1.99 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 3.9850 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0231 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 3.9846 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9828 +num rollout transitions: 50000, reward mean: 3.9777 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 4.0206 +num rollout transitions: 50000, reward mean: 3.9754 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0194 +num rollout transitions: 50000, reward mean: 3.9708 +num rollout transitions: 50000, reward mean: 4.0274 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 4.0035 +---------------------------------------------------------------------------------- +| alpha | 0.136 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 1.69 | +| loss/actor | -340 | +| loss/alpha | -0.00961 | +| loss/critic | 39.4 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 1.96 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0199 +num rollout transitions: 50000, reward mean: 3.9788 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9888 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9798 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 3.9878 +num rollout transitions: 50000, reward mean: 3.9798 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 3.9829 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0063 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 2.11 | +| loss/actor | -347 | +| loss/alpha | -0.0051 | +| loss/critic | 37.8 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 1.95 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 4.0223 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 3.9857 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 3.9828 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 3.9865 +num rollout transitions: 50000, reward mean: 3.9768 +num rollout transitions: 50000, reward mean: 3.9751 +num rollout transitions: 50000, reward mean: 4.0212 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0190 +num rollout transitions: 50000, reward mean: 3.9858 +num rollout transitions: 50000, reward mean: 3.9951 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.675 | +| loss/actor | -350 | +| loss/alpha | -0.00234 | +| loss/critic | 37.7 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 1.94 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9905 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9812 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9994 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 1.91 | +| loss/actor | -345 | +| loss/alpha | 0.00435 | +| loss/critic | 39.3 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 1.98 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 3.9810 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0186 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 3.9813 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 3.9866 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 3.9864 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9837 +num rollout transitions: 50000, reward mean: 3.9871 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 4.0208 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9743 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9761 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0201 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9981 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 1.36 | +| loss/actor | -343 | +| loss/alpha | -0.00065 | +| loss/critic | 39.3 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 1.99 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0184 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9860 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9815 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 3.9825 +num rollout transitions: 50000, reward mean: 3.9716 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 3.9976 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.458 | +| loss/actor | -347 | +| loss/alpha | -0.00305 | +| loss/critic | 39.9 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.02 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9877 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0207 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9883 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9840 +num rollout transitions: 50000, reward mean: 3.9775 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9835 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 3.9743 +num rollout transitions: 50000, reward mean: 3.9789 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 3.9869 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 3.9790 +num rollout transitions: 50000, reward mean: 3.9827 +num rollout transitions: 50000, reward mean: 3.9820 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0231 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 3.9851 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.826 | +| loss/actor | -348 | +| loss/alpha | 0.0033 | +| loss/critic | 41.9 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.06 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 3.9860 +num rollout transitions: 50000, reward mean: 3.9774 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9905 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 3.9812 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 3.9722 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9850 +num rollout transitions: 50000, reward mean: 3.9915 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9672 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 3.9829 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 3.9841 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 3.9831 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9883 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 3.9698 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 3.9704 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 3.9862 +num rollout transitions: 50000, reward mean: 3.9908 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.569 | +| loss/actor | -351 | +| loss/alpha | -0.00257 | +| loss/critic | 40.5 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.02 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 3.99 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9854 +num rollout transitions: 50000, reward mean: 3.9823 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9858 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9743 +num rollout transitions: 50000, reward mean: 3.9790 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 3.9803 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 4.0156 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9867 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 3.9806 +num rollout transitions: 50000, reward mean: 3.9738 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9856 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9827 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.415 | +| loss/actor | -349 | +| loss/alpha | -0.00128 | +| loss/critic | 40.9 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.03 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0156 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 3.9771 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9866 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 3.9823 +num rollout transitions: 50000, reward mean: 3.9758 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9799 +num rollout transitions: 50000, reward mean: 3.9869 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 3.9844 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9779 +num rollout transitions: 50000, reward mean: 3.9735 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 3.9842 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9852 +num rollout transitions: 50000, reward mean: 3.9824 +num rollout transitions: 50000, reward mean: 3.9839 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9757 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0149 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0055 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 969 | +| eval/episode_length_std | 106 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 13.1 | +| loss/actor | -348 | +| loss/alpha | -0.00154 | +| loss/critic | 43.7 | +| penalty/lcb_mean | 1.43 | +| penalty/lcb_std | 2.12 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9849 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9792 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 3.9891 +num rollout transitions: 50000, reward mean: 3.9905 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9840 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 3.9729 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 3.9888 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0234 +num rollout transitions: 50000, reward mean: 4.0065 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 989 | +| eval/episode_length_std | 52.4 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 6.52 | +| loss/actor | -343 | +| loss/alpha | 0.0042 | +| loss/critic | 41.6 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.04 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9865 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0175 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 3.9859 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9791 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9915 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 3.9883 +num rollout transitions: 50000, reward mean: 3.9868 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0107 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.413 | +| loss/actor | -342 | +| loss/alpha | -0.00771 | +| loss/critic | 41.9 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.05 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9765 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9783 +num rollout transitions: 50000, reward mean: 3.9823 +num rollout transitions: 50000, reward mean: 3.9829 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 3.9888 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9699 +num rollout transitions: 50000, reward mean: 3.9798 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9874 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 3.9826 +num rollout transitions: 50000, reward mean: 3.9824 +num rollout transitions: 50000, reward mean: 3.9786 +num rollout transitions: 50000, reward mean: 3.9755 +num rollout transitions: 50000, reward mean: 3.9828 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9787 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 3.9944 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 992 | +| eval/episode_length_std | 47.9 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 5.94 | +| loss/actor | -343 | +| loss/alpha | 0.0105 | +| loss/critic | 42.8 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.1 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0254 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9717 +num rollout transitions: 50000, reward mean: 3.9851 +num rollout transitions: 50000, reward mean: 4.0186 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9808 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9741 +num rollout transitions: 50000, reward mean: 3.9826 +num rollout transitions: 50000, reward mean: 3.9865 +----------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.07 | +| loss/actor | -343 | +| loss/alpha | -0.000569 | +| loss/critic | 45.7 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.16 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1350000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 3.9752 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9794 +num rollout transitions: 50000, reward mean: 3.9834 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 3.9862 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9818 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 3.9837 +num rollout transitions: 50000, reward mean: 4.0296 +num rollout transitions: 50000, reward mean: 3.9835 +num rollout transitions: 50000, reward mean: 4.0190 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9819 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 3.9915 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0205 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 4.0186 +num rollout transitions: 50000, reward mean: 3.9938 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.12 | +| loss/actor | -339 | +| loss/alpha | -0.00386 | +| loss/critic | 47.1 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 4.0232 +num rollout transitions: 50000, reward mean: 4.0195 +num rollout transitions: 50000, reward mean: 4.0221 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0254 +num rollout transitions: 50000, reward mean: 4.0193 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9857 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9871 +num rollout transitions: 50000, reward mean: 3.9809 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9858 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9845 +num rollout transitions: 50000, reward mean: 3.9815 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0246 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 3.9792 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0133 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.913 | +| loss/actor | -338 | +| loss/alpha | 0.00366 | +| loss/critic | 48.9 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.26 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9867 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9733 +num rollout transitions: 50000, reward mean: 3.9834 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9807 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9825 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 3.9708 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0064 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 994 | +| eval/episode_length_std | 33.5 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 4.32 | +| loss/actor | -338 | +| loss/alpha | -0.00261 | +| loss/critic | 49.7 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.27 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0201 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9853 +num rollout transitions: 50000, reward mean: 4.0229 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9861 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0280 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0231 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 3.9811 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 4.0008 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.562 | +| loss/actor | -332 | +| loss/alpha | -0.00108 | +| loss/critic | 49.4 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.25 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9814 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 4.0184 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9722 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0032 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 926 | +| eval/episode_length_std | 156 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 19.5 | +| loss/actor | -333 | +| loss/alpha | 0.00795 | +| loss/critic | 48.4 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9865 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 4.0178 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 3.9789 +num rollout transitions: 50000, reward mean: 3.9821 +num rollout transitions: 50000, reward mean: 3.9743 +num rollout transitions: 50000, reward mean: 3.9653 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9851 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9743 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9813 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0006 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 847 | +| eval/episode_length_std | 210 | +| eval/normalized_episode_reward | 94.5 | +| eval/normalized_episode_reward_std | 25.1 | +| loss/actor | -334 | +| loss/alpha | -0.00384 | +| loss/critic | 50 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.26 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 3.9845 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 3.9902 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9891 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0235 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0215 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9865 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0092 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 990 | +| eval/episode_length_std | 57.3 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 7.28 | +| loss/actor | -338 | +| loss/alpha | 0.00466 | +| loss/critic | 48.5 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0219 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 3.9891 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 3.9748 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9891 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9837 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 3.9860 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9854 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0252 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 3.9769 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9878 +num rollout transitions: 50000, reward mean: 4.0057 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 1.01 | +| loss/actor | -335 | +| loss/alpha | -0.00059 | +| loss/critic | 52.6 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 3.9866 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 3.9796 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 3.9844 +num rollout transitions: 50000, reward mean: 4.0177 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 3.9820 +num rollout transitions: 50000, reward mean: 4.0178 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9844 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 3.9847 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9853 +----------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.893 | +| loss/actor | -336 | +| loss/alpha | -0.000408 | +| loss/critic | 49.8 | +| penalty/lcb_mean | 1.56 | +| penalty/lcb_std | 2.26 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1800000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9825 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 3.9868 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9655 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9778 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 3.9862 +num rollout transitions: 50000, reward mean: 3.9758 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 3.9743 +num rollout transitions: 50000, reward mean: 3.9883 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 3.9822 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9975 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0.696 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 1.16 | +| loss/actor | -335 | +| loss/alpha | 0.00361 | +| loss/critic | 52 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 3.99 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9803 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0172 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9706 +num rollout transitions: 50000, reward mean: 3.9800 +num rollout transitions: 50000, reward mean: 3.9836 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9866 +num rollout transitions: 50000, reward mean: 3.9842 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9826 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9772 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9864 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9960 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.913 | +| loss/actor | -335 | +| loss/alpha | -0.00822 | +| loss/critic | 52.8 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9840 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 3.9823 +num rollout transitions: 50000, reward mean: 4.0199 +num rollout transitions: 50000, reward mean: 3.9803 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9814 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 3.9905 +num rollout transitions: 50000, reward mean: 3.9805 +num rollout transitions: 50000, reward mean: 3.9846 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0335 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0209 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9853 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0000 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.923 | +| loss/actor | -342 | +| loss/alpha | 0.00894 | +| loss/critic | 54.1 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.36 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9837 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9787 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9684 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 3.9842 +num rollout transitions: 50000, reward mean: 3.9801 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 3.9849 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 3.9947 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.39 | +| loss/actor | -338 | +| loss/alpha | -0.00261 | +| loss/critic | 54.3 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9854 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9856 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 3.9876 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0156 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9827 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 3.9838 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9975 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.476 | +| loss/actor | -339 | +| loss/alpha | 0.00156 | +| loss/critic | 54.4 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.36 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9836 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 3.9832 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9877 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9814 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0274 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9855 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 3.9767 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 3.9845 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 3.9815 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9822 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 0.563 | +| loss/actor | -339 | +| loss/alpha | -0.00884 | +| loss/critic | 53 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.3 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9851 +num rollout transitions: 50000, reward mean: 3.9874 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 3.9902 +num rollout transitions: 50000, reward mean: 4.0199 +num rollout transitions: 50000, reward mean: 4.0131 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 146 | +| eval/episode_length_std | 188 | +| eval/normalized_episode_reward | 6.15 | +| eval/normalized_episode_reward_std | 20.1 | +| loss/actor | -338 | +| loss/alpha | 0.0161 | +| loss/critic | 53.2 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0291 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9841 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 3.9840 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 3.9730 +num rollout transitions: 50000, reward mean: 3.9732 +num rollout transitions: 50000, reward mean: 3.9874 +num rollout transitions: 50000, reward mean: 3.9864 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 3.9915 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0060 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 1.22 | +| loss/actor | -340 | +| loss/alpha | -0.00601 | +| loss/critic | 53.8 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9799 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 3.9858 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9844 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 3.9797 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9690 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9967 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 153 | +| eval/episode_length_std | 268 | +| eval/normalized_episode_reward | 10.7 | +| eval/normalized_episode_reward_std | 30.7 | +| loss/actor | -338 | +| loss/alpha | -0.00643 | +| loss/critic | 53.7 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 3.9809 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9876 +num rollout transitions: 50000, reward mean: 3.9798 +num rollout transitions: 50000, reward mean: 3.9800 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 3.9867 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9773 +num rollout transitions: 50000, reward mean: 3.9740 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 3.9856 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0191 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0033 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 996 | +| eval/episode_length_std | 34.4 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 4.11 | +| loss/actor | -339 | +| loss/alpha | -0.00396 | +| loss/critic | 54.2 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9883 +num rollout transitions: 50000, reward mean: 3.9853 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9790 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9871 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 3.9870 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 995 | +| eval/episode_length_std | 32.2 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 3.94 | +| loss/actor | -339 | +| loss/alpha | -0.00147 | +| loss/critic | 53.4 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9862 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 3.9794 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 3.9845 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9832 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 3.9929 +---------------------------------------------------------------------------------- +| alpha | 0.118 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 1.11 | +| loss/actor | -338 | +| loss/alpha | 0.00301 | +| loss/critic | 52.9 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 3.9861 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 3.9823 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 3.9864 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 3.9799 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 3.9832 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 3.9822 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 3.9836 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 865 | +| eval/episode_length_std | 335 | +| eval/normalized_episode_reward | 95.3 | +| eval/normalized_episode_reward_std | 38.2 | +| loss/actor | -339 | +| loss/alpha | -0.00454 | +| loss/critic | 51.9 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9814 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9805 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 3.9836 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9799 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9867 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 3.9994 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 1.09 | +| loss/actor | -336 | +| loss/alpha | 0.0178 | +| loss/critic | 54.2 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 3.9859 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 3.9922 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 990 | +| eval/episode_length_std | 72 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 8.88 | +| loss/actor | -338 | +| loss/alpha | -0.0247 | +| loss/critic | 55.3 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9777 +num rollout transitions: 50000, reward mean: 3.9874 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 3.9857 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9871 +num rollout transitions: 50000, reward mean: 3.9834 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9851 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9862 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 3.9986 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 952 | +| eval/episode_length_std | 210 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 24 | +| loss/actor | -342 | +| loss/alpha | 0.0149 | +| loss/critic | 53.3 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.37 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9883 +num rollout transitions: 50000, reward mean: 3.9808 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9874 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9820 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9715 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9829 +num rollout transitions: 50000, reward mean: 3.9762 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0056 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 980 | +| eval/episode_length_std | 79.2 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 9.55 | +| loss/actor | -341 | +| loss/alpha | 0.0015 | +| loss/critic | 57.3 | +| penalty/lcb_mean | 1.69 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9878 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9864 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9827 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 3.9869 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9830 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 3.9856 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9953 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.659 | +| loss/actor | -342 | +| loss/alpha | 0.00188 | +| loss/critic | 55.8 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.44 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 3.9839 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9846 +num rollout transitions: 50000, reward mean: 3.9814 +num rollout transitions: 50000, reward mean: 3.9858 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9797 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 3.9838 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 3.9809 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0153 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 970 | +| eval/episode_length_std | 101 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 12 | +| loss/actor | -340 | +| loss/alpha | -0.00667 | +| loss/critic | 56.2 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9868 +num rollout transitions: 50000, reward mean: 3.9859 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 3.9853 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 3.9823 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9915 +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 3.9964 +---------------------------------------------------------------------------------- +| alpha | 0.118 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.987 | +| loss/actor | -335 | +| loss/alpha | 0.00581 | +| loss/critic | 57 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0213 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 3.9882 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 4.0230 +num rollout transitions: 50000, reward mean: 4.0213 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0060 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.325 | +| loss/actor | -338 | +| loss/alpha | -0.0253 | +| loss/critic | 56.8 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 3.9833 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 3.9854 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 4.0196 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9957 +---------------------------------------------------------------------------------- +| alpha | 0.104 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 1.91 | +| loss/actor | -334 | +| loss/alpha | 0.045 | +| loss/critic | 51.9 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9874 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9977 +---------------------------------------------------------------------------------- +| alpha | 0.104 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.558 | +| loss/actor | -333 | +| loss/alpha | -0.0295 | +| loss/critic | 52.9 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9835 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 3.9856 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 3.9933 +---------------------------------------------------------------------------------- +| alpha | 0.102 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.474 | +| loss/actor | -333 | +| loss/alpha | 0.0154 | +| loss/critic | 50.2 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 122456.06s diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_42×tamp_26-0317-010307/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_42×tamp_26-0317-010307/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_42×tamp_26-0317-010307/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_42×tamp_26-0317-010307/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..54133ed1905a9879b24cb319d963494d57a9c440 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_42×tamp_26-0317-010307/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_diffusion.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 42, "device": "cuda:7", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_42×tamp_26-0112-030600/model", "reward_penalty_coef": 0.5, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/diffusion/checkpoint.pt", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "density_model": "diffusion", "data_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "penalty_type": "tanh", "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "diffusion"} diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_42×tamp_26-0317-010307/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_42×tamp_26-0317-010307/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..460e922441b35428425f5a63c5c8e66f0be1afa7 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_42×tamp_26-0317-010307/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.26010656816497374,130.97,24.114499787472266,1.9107946711164927,0.7784987379282658,-256.48264065817926,-1.8958086312742153,18.606618245362963,0.9612606417499877,1.1661767950377182,1.0,0.0,50000.0,4.021955242688703,50000 +0.12709427481070423,233.01,114.33735128994375,12.9821679925894,11.479801933810789,-379.77036995177446,0.02125939526477782,30.446741581420834,1.204534055905348,1.6406263143014927,1.0,0.0,50000.0,4.0027834624928476,100000 +0.1279509038324653,182.12,67.7159183648867,2.998862083150013,4.296292259228584,-392.740255734866,-0.006952416578680279,28.905837470569168,1.178120815519095,1.593985981031662,1.0,0.0,50000.0,4.01339224404918,150000 +0.12717082333028182,198.46,80.71560196145477,11.783499229650518,9.15921467462922,-392.10890768066673,0.013588794149980018,31.08840963434213,1.219144772214893,1.654926314842698,1.0,0.0,50000.0,4.012562202227519,200000 +0.14567385011256218,273.56,226.7338227966882,21.217995388321672,23.72778260814971,-372.5295784979237,0.013916833381466607,34.21339832981085,1.2515244646107935,1.728491901484743,1.0,0.0,50000.0,4.015840997724066,250000 +0.1472665258923199,377.82,336.8627429681116,32.64595140394867,35.471024153994115,-358.86687038756935,-0.007750806069448528,33.200788448467264,1.2502425534212476,1.6869028328037348,1.0,0.0,50000.0,4.01636232514032,300000 +0.14924808612912968,134.96,19.0845068052596,7.4280172077292095,1.6289606598511914,-351.17602536559394,0.012815461519844763,34.60595193067573,1.271689899456515,1.7170259440660522,1.0,0.0,50000.0,4.01108395059955,350000 +0.15123358589649416,995.77,42.08796858961003,101.26526088312058,5.007108020126771,-343.45169025084687,-0.013787722290531176,37.346228884067685,1.2946170709454903,1.8175066920149234,1.0,0.0,50000.0,4.00761921748948,400000 +0.14454470500201105,989.45,60.549050364146915,101.99816921185207,6.978080356846632,-341.35771314026067,0.0005468935789540512,39.12895918750812,1.3252389087021383,1.925208973419667,1.0,0.0,50000.0,4.003040370088375,450000 +0.1478827886152271,723.98,258.1679290694334,73.39937790513412,28.988688686390677,-339.60747888794305,-0.0001594891105592554,40.51335264619944,1.3364176510166936,1.934344494587186,1.0,0.0,50000.0,4.008295536635577,500000 +0.14400746827810923,965.21,102.62439232463208,104.77737340457536,12.716643090276925,-344.3077157745326,-0.004483253823295286,37.445861756286895,1.298777192748757,1.8602452720439397,1.0,0.0,50000.0,4.006543300084448,550000 +0.13918747541665924,990.9,53.07909192893186,108.86117268169721,6.903552376571419,-346.96548370605535,-0.001765071741938576,40.038296211776796,1.3318933718860395,1.9624548308229421,1.0,0.0,50000.0,4.0005600417546034,600000 +0.14165124107926955,473.86,403.255453031946,46.8536497715013,46.890288636983314,-342.0764211480738,0.008073747935406834,41.784936601429,1.3658301079177957,1.9894073920834079,1.0,0.0,50000.0,4.000735688910414,650000 +0.1357803334930541,1000.0,0.0,115.19158706264272,1.692130717389854,-340.2869052252231,-0.009607978770323199,39.38201594957346,1.3194681610786814,1.9608220427525018,1.0,0.0,50000.0,4.001061419050802,700000 +0.1246116629639268,1000.0,0.0,107.84282082184032,2.1085733182480255,-346.9877406072999,-0.005104628092627872,37.81227524908145,1.304365783603172,1.95215701367856,1.0,0.0,50000.0,4.000648028172889,750000 +0.123957433483601,1000.0,0.0,113.3255649779609,0.6753494315056687,-350.206260969238,-0.002344601191058732,37.74622146785697,1.3004681524717814,1.9354510277390704,1.0,0.0,50000.0,4.000218178480686,800000 +0.12799933589115675,1000.0,0.0,114.7645690386286,1.9088709358569822,-344.75813232849106,0.004352000586017901,39.32933093364716,1.3273272403335512,1.981255337963123,1.0,0.0,50000.0,4.001950874340638,850000 +0.1277914002023633,1000.0,0.0,115.88941916837227,1.363350338025509,-343.0662506060798,-0.0006496161188185533,39.25816836275125,1.3317254096651088,1.9888164294183144,1.0,0.0,50000.0,3.999528294739295,900000 +0.12376643388584449,1000.0,0.0,113.17793099874682,0.45823820140721033,-347.4316422253414,-0.003047418961338718,39.898947081604604,1.3405836998641305,2.019219444791048,1.0,0.0,50000.0,3.999576288271784,950000 +0.12864695900216477,1000.0,0.0,110.01019047931149,0.8258539673208073,-347.98238498596317,0.0032970714040844703,41.85238346820817,1.4033183644604668,2.055191384930586,1.0,0.0,50000.0,3.99688889936607,1000000 +0.12278977039709718,1000.0,0.0,108.95542978820933,0.5690550356577393,-351.3434851507586,-0.0025740299267695255,40.48551656776398,1.4020658664393388,2.019009943474559,1.0,0.0,50000.0,3.9931075655484554,1050000 +0.12469853136807628,1000.0,0.0,109.87376436744832,0.41494865915508927,-349.16342880554896,-0.0012755583744494786,40.91930036977736,1.3916229855644586,2.0309456380307664,1.0,0.0,50000.0,3.995965884254742,1100000 +0.1232311908322579,969.22,106.49277721986594,108.60297737355877,13.07905925790534,-347.61781209899897,-0.0015413823386654796,43.71490368328097,1.4308925750362693,2.1228589208281083,1.0,0.0,50000.0,3.995035174887871,1150000 +0.12755881536871247,988.65,52.42830819318892,110.79078915069809,6.519153147291503,-342.88411621337553,0.004202031796462662,41.606132142086494,1.4141767189705663,2.0442072276771124,1.0,0.0,50000.0,3.999666119725298,1200000 +0.11901343632236125,1000.0,0.0,110.42819864906201,0.4125037429072046,-341.8416523071297,-0.007714420828260286,41.868383156833836,1.4024333186602531,2.0491509804344226,1.0,0.0,50000.0,3.9988018871844884,1250000 +0.11887223177000887,992.37,47.91401778185586,111.81538487383851,5.939872192768089,-342.92795847900584,0.0105012957952916,42.781525374642186,1.410510551500316,2.102682868096833,1.0,0.0,50000.0,3.9956608577879424,1300000 +0.12274735473215531,1000.0,0.0,113.65569232296642,1.0668619351000062,-342.8511651904324,-0.0005693821896611716,45.680143537367336,1.460931850658635,2.160841324335328,1.0,0.0,50000.0,3.9983393918661347,1350000 +0.12685425558656535,1000.0,0.0,113.06430826781654,1.1158683659593727,-338.8936623413095,-0.0038585419637710053,47.11997244182565,1.4981153171205497,2.205286464070061,1.0,0.0,50000.0,3.9980962764844414,1400000 +0.12383224204018745,1000.0,0.0,116.42655097012164,0.9128709607782322,-338.10407191649676,0.0036597150060534668,48.943963979224655,1.508226298632619,2.2616171534037828,1.0,0.0,50000.0,4.001258044909991,1450000 +0.12762682418466098,994.22,33.537018352858986,112.66499877990816,4.322345005466686,-337.9138551818906,-0.002605840455740658,49.735978810863685,1.5376809922993255,2.26798851660373,1.0,0.0,50000.0,3.9983790697944284,1500000 +0.12828904828503815,1000.0,0.0,113.40269996965321,0.5618555679554799,-332.0974151123079,-0.0010847443978861178,49.43192394853594,1.5235914594960187,2.2512906406521966,1.0,0.0,50000.0,4.001514951800789,1550000 +0.1268514621634788,926.15,155.6354313773056,105.19440205315735,19.505754143689295,-332.77767801269806,0.007946124373413587,48.4382926787753,1.5238565038192065,2.2238645958137564,1.0,0.0,50000.0,4.001721770384169,1600000 +0.12874401711791614,847.1,209.62001335750367,94.45218135748587,25.11584333812742,-333.65032450805677,-0.003835182695016084,50.01518024585817,1.570685843701341,2.26420511926534,1.0,0.0,50000.0,3.9997354018454314,1650000 +0.12462082088455619,990.13,57.25428455583042,115.16529385973952,7.280046826191946,-337.81561410400286,0.004660705312863097,48.512494826888464,1.5248604657244729,2.2213533751976287,1.0,0.0,50000.0,4.0016922925049325,1700000 +0.12736922225266595,1000.0,0.0,111.54048337461506,1.009874271265403,-335.06128598755356,-0.0005895134041831382,52.59365942489681,1.5886270876503032,2.3195078652608467,1.0,0.0,50000.0,4.001327255772077,1750000 +0.12227996642723785,1000.0,0.0,109.44720925766104,0.893086924372804,-335.5823021276835,-0.00040804778542366456,49.81934460487312,1.557938111605649,2.263447037438146,1.0,0.0,50000.0,3.9991080309797296,1800000 +0.12606504004523297,999.93,0.6964912059746339,112.00719625377029,1.1554844219517315,-335.00227744873524,0.0036083246730266054,51.979072444019046,1.5819936350083346,2.3159892958796036,1.0,0.0,50000.0,3.994945747715712,1850000 +0.12639163313865737,1000.0,0.0,114.47458447916618,0.9126658767315776,-334.71203083373973,-0.00821855211783194,52.84185580846796,1.5979908638930245,2.3210558021783934,1.0,0.0,50000.0,3.995280855806231,1900000 +0.13023429559454383,1000.0,0.0,112.34422195944818,0.9226728742777768,-341.85745468322506,0.008938447152786081,54.1183434511187,1.6375159090089884,2.355035180714112,1.0,0.0,50000.0,4.001958490743792,1950000 +0.13316828544512394,1000.0,0.0,112.83392314431889,1.3933685007395413,-338.0643818798805,-0.002608150453679401,54.315259019431174,1.6414713014197369,2.330465105178391,1.0,0.0,50000.0,3.9971866358698502,2000000 +0.13115901021733803,1000.0,0.0,111.29612033744986,0.47619092451239453,-339.1929561914102,0.0015612457858399371,54.40844697027223,1.6408441787218833,2.3640582389760216,1.0,0.0,50000.0,3.9990470491484507,2050000 +0.1252998694384113,1000.0,0.0,117.88450248948557,0.5633958197976473,-338.5571184771746,-0.008835066727437196,52.989682667675204,1.6124678628087012,2.303564701884952,1.0,0.0,50000.0,3.996600558070921,2100000 +0.12970751872822492,145.69,187.61576133150433,6.149801357477698,20.11995821389479,-338.0767801336667,0.016121772066727425,53.15053862445828,1.6172458002495664,2.324132378926295,1.0,0.0,50000.0,4.002813600851859,2150000 +0.13073222350567926,1000.0,0.0,114.91733015649469,1.2223214443316757,-339.51386640869356,-0.0060100440295414006,53.840795952072426,1.6374746879506026,2.349788279447598,1.0,0.0,50000.0,3.9973274050591114,2200000 +0.13064828044995713,153.38,267.6719551988964,10.676060473951491,30.662995548700795,-338.26988545471204,-0.006434891598485412,53.679045811271706,1.63758027415989,2.3462318637037245,1.0,0.0,50000.0,3.999314084457742,2250000 +0.12771819902747944,995.98,34.40726086162628,111.59897552453866,4.1083788047102665,-338.89907398498383,-0.003955865126289462,54.184221485252046,1.6413794755077327,2.3827945502352486,1.0,0.0,50000.0,3.9978663546808115,2300000 +0.12594480560064275,994.78,32.19521082397194,112.1379791679622,3.9370954053494565,-338.5716271807872,-0.0014656031234934908,53.38788651960383,1.6281556152296042,2.3420486280262414,1.0,0.0,50000.0,3.9996213848847506,2350000 +0.11750731855109336,1000.0,0.0,111.69465075691238,1.1088282678382826,-337.6921084295625,0.003008367100655982,52.94142979062997,1.6060709646344389,2.335182682096955,1.0,0.0,50000.0,3.999829846967733,2400000 +0.11675456903323607,864.87,334.91756164763893,95.26082082803964,38.19920322153344,-339.3973001824964,-0.0045388382646441286,51.94149936233503,1.5942507414269194,2.339023500318491,1.0,0.0,50000.0,3.9970981764481066,2450000 +0.12099763264939048,1000.0,0.0,110.12123333097905,1.0907276652047835,-336.04053546447835,0.017798508748300523,54.18572754081667,1.621112435140598,2.3913835898565767,1.0,0.0,50000.0,3.9987014947837474,2500000 +0.12147366511464143,989.72,71.9668090163792,115.17144137521822,8.88255010073947,-337.73905243590934,-0.024715230739377356,55.25614226593027,1.6517033020568004,2.4187892270350413,1.0,0.0,50000.0,3.9993153091658824,2550000 +0.11619932412639328,951.82,210.01197013503779,105.13784312385049,24.004215854811772,-341.6928641577134,0.014892197536826149,53.25883416704087,1.621154416859143,2.3727441968059537,1.0,0.0,50000.0,3.999295208890605,2600000 +0.12562259308978937,979.85,79.17756942467986,111.65099930566618,9.554785939172564,-340.7154783276325,0.0015022562593965312,57.253447132301154,1.6914641883993053,2.4618548676085057,1.0,0.0,50000.0,3.9961951480713482,2650000 +0.11722731672257158,1000.0,0.0,113.59124632688851,0.6591344564596759,-342.26681687683043,0.0018814271019957302,55.80619983457502,1.6579601529908314,2.437173225526802,1.0,0.0,50000.0,3.9967052303848156,2700000 +0.11664808100014948,969.83,101.4875415999422,109.48817785286839,11.957425426865765,-340.21018974242867,-0.006669060201644771,56.20948833145164,1.660449171569346,2.446168809027666,1.0,0.0,50000.0,3.996471496861553,2750000 +0.11807767802447008,1000.0,0.0,109.39674075217609,0.9873851040322155,-335.4935671850549,0.005805025798790391,57.0429044169236,1.6728198187303354,2.4605529312038046,1.0,0.0,50000.0,3.997813331750451,2800000 +0.11563184144735272,1000.0,0.0,114.9317730425802,0.32503101156617803,-337.86756456543503,-0.025294752838425534,56.75294603124644,1.6844992979908295,2.4625276932120004,1.0,0.0,50000.0,4.002236844178546,2850000 +0.10427119959667254,1000.0,0.0,115.98537934336296,1.9059086910549323,-333.8112125933808,0.045019993291943984,51.90261201499937,1.5851450817322694,2.3822096522652605,1.0,0.0,50000.0,3.998832019666194,2900000 +0.10417378894537471,1000.0,0.0,113.34238250355514,0.5580029549020875,-333.3483240167251,-0.029490160815902097,52.85087845493313,1.6075413137936483,2.3850693958378235,1.0,0.0,50000.0,3.9994660888032416,2950000 +0.10185466848015769,1000.0,0.0,113.41326237263229,0.47436042946129564,-333.36268919800347,0.0154369046681374,50.23351942893877,1.5713010388541382,2.3391393603169885,1.0,0.0,50000.0,3.997827773994398,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_456×tamp_26-0319-224114/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_456×tamp_26-0319-224114/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..c8b28d8f67646a5e478295e9c4c0910d33460006 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_456×tamp_26-0319-224114/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 3.9105 +num rollout transitions: 50000, reward mean: 4.0258 +num rollout transitions: 50000, reward mean: 4.0735 +num rollout transitions: 50000, reward mean: 4.0537 +num rollout transitions: 50000, reward mean: 4.0609 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 3.9879 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0222 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0291 +num rollout transitions: 50000, reward mean: 4.0349 +num rollout transitions: 50000, reward mean: 4.0411 +num rollout transitions: 50000, reward mean: 4.0325 +num rollout transitions: 50000, reward mean: 4.0382 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0294 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 4.0255 +num rollout transitions: 50000, reward mean: 4.0444 +num rollout transitions: 50000, reward mean: 4.0370 +num rollout transitions: 50000, reward mean: 4.0348 +num rollout transitions: 50000, reward mean: 4.0360 +num rollout transitions: 50000, reward mean: 4.0579 +num rollout transitions: 50000, reward mean: 4.0249 +num rollout transitions: 50000, reward mean: 4.0477 +num rollout transitions: 50000, reward mean: 4.0368 +num rollout transitions: 50000, reward mean: 4.0201 +num rollout transitions: 50000, reward mean: 4.0301 +num rollout transitions: 50000, reward mean: 4.0286 +num rollout transitions: 50000, reward mean: 4.0256 +num rollout transitions: 50000, reward mean: 4.0339 +num rollout transitions: 50000, reward mean: 4.0438 +num rollout transitions: 50000, reward mean: 4.0442 +num rollout transitions: 50000, reward mean: 4.0390 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 4.0179 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0230 +num rollout transitions: 50000, reward mean: 4.0414 +num rollout transitions: 50000, reward mean: 4.0219 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 3.9787 +num rollout transitions: 50000, reward mean: 4.0332 +num rollout transitions: 50000, reward mean: 4.0303 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0149 +---------------------------------------------------------------------------------- +| alpha | 0.255 | +| eval/episode_length | 213 | +| eval/episode_length_std | 85.5 | +| eval/normalized_episode_reward | 10.8 | +| eval/normalized_episode_reward_std | 5.79 | +| loss/actor | -262 | +| loss/alpha | -1.68 | +| loss/critic | 17.5 | +| penalty/lcb_mean | 0.903 | +| penalty/lcb_std | 1.15 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.02 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 3.9847 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 3.9741 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0183 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9847 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0311 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0318 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0039 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 132 | +| eval/episode_length_std | 93.4 | +| eval/normalized_episode_reward | 3.65 | +| eval/normalized_episode_reward_std | 7.86 | +| loss/actor | -384 | +| loss/alpha | 0.000229 | +| loss/critic | 26 | +| penalty/lcb_mean | 1.06 | +| penalty/lcb_std | 1.57 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 3.9834 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 4.0264 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0217 +num rollout transitions: 50000, reward mean: 4.0209 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0355 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 4.0341 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0193 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 4.0234 +num rollout transitions: 50000, reward mean: 4.0241 +num rollout transitions: 50000, reward mean: 4.0445 +num rollout transitions: 50000, reward mean: 4.0244 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0279 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0261 +num rollout transitions: 50000, reward mean: 4.0220 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0365 +num rollout transitions: 50000, reward mean: 4.0327 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 4.0219 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0245 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 357 | +| eval/episode_length_std | 342 | +| eval/normalized_episode_reward | 28.8 | +| eval/normalized_episode_reward_std | 35.3 | +| loss/actor | -392 | +| loss/alpha | -0.00901 | +| loss/critic | 25.1 | +| penalty/lcb_mean | 1.04 | +| penalty/lcb_std | 1.55 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 4.0227 +num rollout transitions: 50000, reward mean: 4.0222 +num rollout transitions: 50000, reward mean: 4.0312 +num rollout transitions: 50000, reward mean: 4.0267 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 4.0158 +num rollout transitions: 50000, reward mean: 4.0202 +num rollout transitions: 50000, reward mean: 3.9857 +num rollout transitions: 50000, reward mean: 4.0168 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0444 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 4.0268 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0277 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0299 +num rollout transitions: 50000, reward mean: 4.0293 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9866 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0309 +num rollout transitions: 50000, reward mean: 4.0205 +num rollout transitions: 50000, reward mean: 4.0315 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0294 +num rollout transitions: 50000, reward mean: 4.0223 +num rollout transitions: 50000, reward mean: 4.0319 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 213 | +| eval/episode_length_std | 130 | +| eval/normalized_episode_reward | 8.76 | +| eval/normalized_episode_reward_std | 12.1 | +| loss/actor | -388 | +| loss/alpha | 0.0135 | +| loss/critic | 23.9 | +| penalty/lcb_mean | 1.04 | +| penalty/lcb_std | 1.45 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0196 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0235 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0199 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0216 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0241 +num rollout transitions: 50000, reward mean: 4.0189 +num rollout transitions: 50000, reward mean: 4.0306 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0203 +num rollout transitions: 50000, reward mean: 4.0184 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0274 +num rollout transitions: 50000, reward mean: 4.0195 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 4.0156 +num rollout transitions: 50000, reward mean: 4.0198 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 4.0251 +num rollout transitions: 50000, reward mean: 4.0233 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0252 +num rollout transitions: 50000, reward mean: 4.0220 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0206 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0217 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0321 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0207 +num rollout transitions: 50000, reward mean: 4.0336 +num rollout transitions: 50000, reward mean: 4.0153 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 221 | +| eval/episode_length_std | 242 | +| eval/normalized_episode_reward | 15.6 | +| eval/normalized_episode_reward_std | 25.6 | +| loss/actor | -375 | +| loss/alpha | 0.00203 | +| loss/critic | 27.2 | +| penalty/lcb_mean | 1.09 | +| penalty/lcb_std | 1.53 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0210 +num rollout transitions: 50000, reward mean: 4.0217 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0219 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0260 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0199 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 4.0298 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0277 +num rollout transitions: 50000, reward mean: 4.0306 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0279 +num rollout transitions: 50000, reward mean: 4.0213 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9867 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0186 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 4.0213 +num rollout transitions: 50000, reward mean: 4.0301 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0189 +num rollout transitions: 50000, reward mean: 4.0118 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 603 | +| eval/episode_length_std | 409 | +| eval/normalized_episode_reward | 55.5 | +| eval/normalized_episode_reward_std | 44.1 | +| loss/actor | -362 | +| loss/alpha | -0.00187 | +| loss/critic | 32.5 | +| penalty/lcb_mean | 1.2 | +| penalty/lcb_std | 1.68 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0263 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0248 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0226 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0302 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0178 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 3.9823 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9810 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 4.0156 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 3.9775 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0218 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9905 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9837 +num rollout transitions: 50000, reward mean: 4.0132 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 935 | +| eval/episode_length_std | 131 | +| eval/normalized_episode_reward | 94.6 | +| eval/normalized_episode_reward_std | 14.3 | +| loss/actor | -357 | +| loss/alpha | -0.00304 | +| loss/critic | 32.5 | +| penalty/lcb_mean | 1.2 | +| penalty/lcb_std | 1.68 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 3.9862 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9791 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0224 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0207 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9883 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0188 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0183 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 958 | +| eval/episode_length_std | 110 | +| eval/normalized_episode_reward | 95.5 | +| eval/normalized_episode_reward_std | 11.6 | +| loss/actor | -358 | +| loss/alpha | 0.00557 | +| loss/critic | 33.5 | +| penalty/lcb_mean | 1.19 | +| penalty/lcb_std | 1.71 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9859 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0027 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 872 | +| eval/episode_length_std | 203 | +| eval/normalized_episode_reward | 88.3 | +| eval/normalized_episode_reward_std | 21.4 | +| loss/actor | -360 | +| loss/alpha | -0.012 | +| loss/critic | 34.1 | +| penalty/lcb_mean | 1.2 | +| penalty/lcb_std | 1.78 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0288 +num rollout transitions: 50000, reward mean: 3.9858 +num rollout transitions: 50000, reward mean: 3.9858 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 3.9871 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 3.9723 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0251 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9830 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0023 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 2.15 | +| loss/actor | -363 | +| loss/alpha | -0.00568 | +| loss/critic | 34.5 | +| penalty/lcb_mean | 1.19 | +| penalty/lcb_std | 1.83 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 3.9847 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 3.9836 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0231 +num rollout transitions: 50000, reward mean: 4.0156 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0214 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0225 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 4.0229 +num rollout transitions: 50000, reward mean: 4.0227 +num rollout transitions: 50000, reward mean: 4.0245 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0310 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0050 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 939 | +| eval/episode_length_std | 121 | +| eval/normalized_episode_reward | 97.8 | +| eval/normalized_episode_reward_std | 14.2 | +| loss/actor | -361 | +| loss/alpha | 0.00957 | +| loss/critic | 34.1 | +| penalty/lcb_mean | 1.2 | +| penalty/lcb_std | 1.82 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9866 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 4.0212 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 3.9877 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0178 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 4.0175 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 3.9888 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9783 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 3.9867 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9840 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 4.0193 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0067 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 983 | +| eval/episode_length_std | 66 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 7.86 | +| loss/actor | -363 | +| loss/alpha | -0.00493 | +| loss/critic | 37.7 | +| penalty/lcb_mean | 1.25 | +| penalty/lcb_std | 1.93 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 3.9827 +num rollout transitions: 50000, reward mean: 3.9869 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 4.0206 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0223 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9879 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0298 +num rollout transitions: 50000, reward mean: 3.9958 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 924 | +| eval/episode_length_std | 178 | +| eval/normalized_episode_reward | 93.2 | +| eval/normalized_episode_reward_std | 20.3 | +| loss/actor | -364 | +| loss/alpha | -0.00197 | +| loss/critic | 35.7 | +| penalty/lcb_mean | 1.22 | +| penalty/lcb_std | 1.89 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9854 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 3.9797 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 3.9811 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 3.9758 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9813 +num rollout transitions: 50000, reward mean: 3.9877 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0239 +num rollout transitions: 50000, reward mean: 4.0272 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0175 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9843 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 3.9857 +num rollout transitions: 50000, reward mean: 3.9992 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 964 | +| eval/episode_length_std | 91 | +| eval/normalized_episode_reward | 98.1 | +| eval/normalized_episode_reward_std | 10.6 | +| loss/actor | -365 | +| loss/alpha | -0.00132 | +| loss/critic | 38.3 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.97 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 3.9856 +num rollout transitions: 50000, reward mean: 3.9739 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9809 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 3.9833 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 3.9833 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9762 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9871 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9807 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0168 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9891 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 997 | +| eval/episode_length_std | 30.5 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 4.52 | +| loss/actor | -364 | +| loss/alpha | 0.00927 | +| loss/critic | 38.3 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 1.97 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9853 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 3.9839 +num rollout transitions: 50000, reward mean: 3.9882 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0231 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9888 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9821 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9811 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 3.9754 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9791 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9838 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 3.9848 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0275 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 1.18 | +| loss/actor | -361 | +| loss/alpha | -0.00771 | +| loss/critic | 38.2 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 1.99 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 3.9837 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0259 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9856 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9787 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 3.9891 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 3.9835 +num rollout transitions: 50000, reward mean: 3.9765 +num rollout transitions: 50000, reward mean: 3.9854 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9813 +num rollout transitions: 50000, reward mean: 4.0070 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 948 | +| eval/episode_length_std | 110 | +| eval/normalized_episode_reward | 99.5 | +| eval/normalized_episode_reward_std | 12.8 | +| loss/actor | -365 | +| loss/alpha | -0.00763 | +| loss/critic | 38.1 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 2.01 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9787 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0267 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 3.9876 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9902 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 3.9867 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 3.9780 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 3.9802 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 0.923 | +| loss/actor | -366 | +| loss/alpha | 0.00526 | +| loss/critic | 39 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 2.05 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9882 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0190 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 3.9796 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9848 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 3.9838 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 4.0354 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0078 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.35 | +| loss/actor | -365 | +| loss/alpha | 0.00389 | +| loss/critic | 39.6 | +| penalty/lcb_mean | 1.28 | +| penalty/lcb_std | 2.07 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0223 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0175 +num rollout transitions: 50000, reward mean: 4.0254 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 3.9761 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 4.0181 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 3.9855 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 3.9771 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 3.9832 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9811 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 994 | +| eval/episode_length_std | 31.9 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 3.88 | +| loss/actor | -366 | +| loss/alpha | -0.00136 | +| loss/critic | 38.5 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 2.01 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9782 +num rollout transitions: 50000, reward mean: 4.0218 +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 3.9851 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9751 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0149 +num rollout transitions: 50000, reward mean: 3.9782 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0201 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0015 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 984 | +| eval/episode_length_std | 77.8 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 9.15 | +| loss/actor | -364 | +| loss/alpha | 0.00168 | +| loss/critic | 40 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 2.1 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9718 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 3.9829 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9866 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0206 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9840 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0022 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 1.08 | +| loss/actor | -363 | +| loss/alpha | -0.00826 | +| loss/critic | 38 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 2.02 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 3.9767 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 4.0196 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 3.9850 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 3.9878 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9843 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9829 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9891 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9714 +num rollout transitions: 50000, reward mean: 3.9900 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 1.65 | +| loss/actor | -363 | +| loss/alpha | 0.00409 | +| loss/critic | 40.7 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.08 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 3.9842 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9882 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 4.0233 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0301 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 3.9786 +num rollout transitions: 50000, reward mean: 3.9737 +num rollout transitions: 50000, reward mean: 3.9844 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 3.9792 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 3.9865 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9851 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 3.9826 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 3.9825 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0171 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 3.9846 +num rollout transitions: 50000, reward mean: 3.9949 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 988 | +| eval/episode_length_std | 65.3 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 7.95 | +| loss/actor | -361 | +| loss/alpha | 0.00874 | +| loss/critic | 40.3 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 2.08 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 3.9664 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 3.9850 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 4.0198 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9874 +num rollout transitions: 50000, reward mean: 3.9836 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 3.9758 +num rollout transitions: 50000, reward mean: 3.9820 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9813 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9854 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 3.9775 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 3.9801 +num rollout transitions: 50000, reward mean: 3.9860 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9958 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 993 | +| eval/episode_length_std | 45.3 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 5.88 | +| loss/actor | -364 | +| loss/alpha | -0.0122 | +| loss/critic | 38.5 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 2.03 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0185 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0158 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 3.9646 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9856 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9776 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0281 +num rollout transitions: 50000, reward mean: 3.9888 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0218 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 997 | +| eval/episode_length_std | 29.8 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 3.98 | +| loss/actor | -363 | +| loss/alpha | 0.011 | +| loss/critic | 38.3 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 2.05 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 3.9826 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9905 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9926 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0183 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9832 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 3.9791 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 3.9831 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 4.0166 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 2.97 | +| loss/actor | -362 | +| loss/alpha | -0.00278 | +| loss/critic | 39.4 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 2.11 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9740 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 4.0207 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9807 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9850 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 3.9811 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 958 | +| eval/episode_length_std | 124 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 14.6 | +| loss/actor | -361 | +| loss/alpha | 0.00692 | +| loss/critic | 39.6 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 2.11 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9825 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9857 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0200 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0207 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 3.9857 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0223 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0020 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 2.09 | +| loss/actor | -359 | +| loss/alpha | 0.00678 | +| loss/critic | 40.4 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 2.1 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 3.9800 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0055 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0186 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0134 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0211 +num rollout transitions: 50000, reward mean: 3.9876 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 4.0157 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9871 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0201 +num rollout transitions: 50000, reward mean: 3.9968 +---------------------------------------------------------------------------------- +| alpha | 0.118 | +| eval/episode_length | 994 | +| eval/episode_length_std | 51.5 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 6.51 | +| loss/actor | -359 | +| loss/alpha | -0.0129 | +| loss/critic | 40.7 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.12 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9823 +num rollout transitions: 50000, reward mean: 4.0206 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0151 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9828 +num rollout transitions: 50000, reward mean: 3.9819 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 3.9816 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 4.0139 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9833 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 4.0252 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0136 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 936 | +| eval/episode_length_std | 146 | +| eval/normalized_episode_reward | 97.3 | +| eval/normalized_episode_reward_std | 16.2 | +| loss/actor | -359 | +| loss/alpha | 0.00986 | +| loss/critic | 44.2 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.2 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9778 +num rollout transitions: 50000, reward mean: 3.9748 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 3.9834 +num rollout transitions: 50000, reward mean: 4.0165 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 3.9902 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9915 +num rollout transitions: 50000, reward mean: 3.9898 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9762 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9770 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9870 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 1.11 | +| loss/actor | -359 | +| loss/alpha | -0.00964 | +| loss/critic | 44.2 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0263 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 4.0208 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 3.9855 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0305 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 3.9799 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0228 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.707 | +| loss/actor | -361 | +| loss/alpha | 0.00615 | +| loss/critic | 43.8 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.25 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9859 +num rollout transitions: 50000, reward mean: 3.9820 +num rollout transitions: 50000, reward mean: 3.9847 +num rollout transitions: 50000, reward mean: 3.9794 +num rollout transitions: 50000, reward mean: 3.9877 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9861 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9815 +num rollout transitions: 50000, reward mean: 3.9836 +num rollout transitions: 50000, reward mean: 3.9853 +num rollout transitions: 50000, reward mean: 3.9854 +num rollout transitions: 50000, reward mean: 3.9775 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 3.9843 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9784 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0151 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.505 | +| loss/actor | -358 | +| loss/alpha | -0.00242 | +| loss/critic | 42.8 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.2 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 3.9874 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0267 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 3.9800 +num rollout transitions: 50000, reward mean: 3.9897 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0213 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 3.9824 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0177 +num rollout transitions: 50000, reward mean: 3.9810 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9821 +num rollout transitions: 50000, reward mean: 3.9798 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0156 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 4.0228 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 3.9883 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 3.9943 +----------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 1.06 | +| loss/actor | -359 | +| loss/alpha | -0.000202 | +| loss/critic | 43.7 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.23 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1750000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9745 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 4.0172 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0191 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 3.9893 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 3.9808 +num rollout transitions: 50000, reward mean: 3.9725 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 4.0263 +num rollout transitions: 50000, reward mean: 3.9875 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 3.9740 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 4.0188 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0189 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0194 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 993 | +| eval/episode_length_std | 40.4 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 5.21 | +| loss/actor | -356 | +| loss/alpha | -0.0026 | +| loss/critic | 41.7 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.18 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0311 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0213 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0120 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0156 +num rollout transitions: 50000, reward mean: 4.0282 +num rollout transitions: 50000, reward mean: 4.0363 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0250 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0196 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0150 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0189 +num rollout transitions: 50000, reward mean: 4.0114 +num rollout transitions: 50000, reward mean: 4.0268 +num rollout transitions: 50000, reward mean: 4.0176 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0156 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0279 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 4.0143 +---------------------------------------------------------------------------------- +| alpha | 0.115 | +| eval/episode_length | 995 | +| eval/episode_length_std | 54.6 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 7.09 | +| loss/actor | -361 | +| loss/alpha | 0.000583 | +| loss/critic | 40.9 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 2.16 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9830 +num rollout transitions: 50000, reward mean: 3.9833 +num rollout transitions: 50000, reward mean: 3.9869 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0217 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0163 +num rollout transitions: 50000, reward mean: 4.0197 +num rollout transitions: 50000, reward mean: 4.0141 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 4.0206 +num rollout transitions: 50000, reward mean: 4.0209 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9840 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0076 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9902 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 3.9970 +---------------------------------------------------------------------------------- +| alpha | 0.113 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 1.02 | +| loss/actor | -360 | +| loss/alpha | -0.00164 | +| loss/critic | 43.2 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0183 +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9858 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 4.0148 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 3.9837 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0256 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0234 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0159 +num rollout transitions: 50000, reward mean: 4.0191 +num rollout transitions: 50000, reward mean: 4.0220 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9882 +num rollout transitions: 50000, reward mean: 4.0143 +num rollout transitions: 50000, reward mean: 4.0159 +---------------------------------------------------------------------------------- +| alpha | 0.113 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 2.4 | +| loss/actor | -359 | +| loss/alpha | 0.00946 | +| loss/critic | 41.4 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 2.2 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0284 +num rollout transitions: 50000, reward mean: 4.0124 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0158 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9860 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0222 +num rollout transitions: 50000, reward mean: 4.0233 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9966 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0282 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0245 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 4.0022 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 996 | +| eval/episode_length_std | 27.2 | +| eval/normalized_episode_reward | 120 | +| eval/normalized_episode_reward_std | 3.97 | +| loss/actor | -359 | +| loss/alpha | -0.00168 | +| loss/critic | 43.5 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 2.2 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0176 +num rollout transitions: 50000, reward mean: 4.0278 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0209 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0213 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0249 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9809 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 3.9972 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.863 | +| loss/actor | -358 | +| loss/alpha | -0.00869 | +| loss/critic | 43.4 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.2 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0180 +num rollout transitions: 50000, reward mean: 4.0245 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0268 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 3.9856 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0230 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0217 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0230 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0101 +num rollout transitions: 50000, reward mean: 3.9902 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9915 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0032 +----------------------------------------------------------------------------------- +| alpha | 0.109 | +| eval/episode_length | 968 | +| eval/episode_length_std | 101 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 11.9 | +| loss/actor | -362 | +| loss/alpha | -0.000595 | +| loss/critic | 43.7 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2100000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 3.9905 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 3.9829 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0185 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0149 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0189 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 4.0243 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 4.0205 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 3.9996 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 119 | +| eval/normalized_episode_reward_std | 0.528 | +| loss/actor | -358 | +| loss/alpha | 0.00883 | +| loss/critic | 46.6 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.3 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0166 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0285 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 3.9778 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0204 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0190 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0050 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0135 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 3.9877 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0158 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0062 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 982 | +| eval/episode_length_std | 83 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 10.9 | +| loss/actor | -358 | +| loss/alpha | 0.00607 | +| loss/critic | 44.3 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.26 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.01 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9914 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9840 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0170 +num rollout transitions: 50000, reward mean: 4.0096 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9970 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9866 +num rollout transitions: 50000, reward mean: 3.9908 +num rollout transitions: 50000, reward mean: 3.9845 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0077 +---------------------------------------------------------------------------------- +| alpha | 0.115 | +| eval/episode_length | 955 | +| eval/episode_length_std | 122 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 14.5 | +| loss/actor | -361 | +| loss/alpha | -0.00552 | +| loss/critic | 47.3 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0083 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 3.9937 +num rollout transitions: 50000, reward mean: 4.0213 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 3.9874 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 4.0130 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 3.9766 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0158 +num rollout transitions: 50000, reward mean: 4.0104 +num rollout transitions: 50000, reward mean: 4.0042 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0254 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 4.0138 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9917 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 3.9860 +num rollout transitions: 50000, reward mean: 3.9925 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0089 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 1.12 | +| loss/actor | -358 | +| loss/alpha | -0.00719 | +| loss/critic | 46.4 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 3.9859 +num rollout transitions: 50000, reward mean: 3.9903 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0147 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0175 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0279 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 3.9994 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9773 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9881 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 3.9985 +num rollout transitions: 50000, reward mean: 3.9867 +num rollout transitions: 50000, reward mean: 3.9904 +num rollout transitions: 50000, reward mean: 4.0020 +---------------------------------------------------------------------------------- +| alpha | 0.111 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 119 | +| eval/normalized_episode_reward_std | 0.832 | +| loss/actor | -359 | +| loss/alpha | -0.0196 | +| loss/critic | 43.8 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9811 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 4.0183 +num rollout transitions: 50000, reward mean: 3.9739 +num rollout transitions: 50000, reward mean: 3.9821 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0125 +num rollout transitions: 50000, reward mean: 4.0126 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 4.0188 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9853 +num rollout transitions: 50000, reward mean: 3.9905 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0117 +num rollout transitions: 50000, reward mean: 3.9844 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0173 +num rollout transitions: 50000, reward mean: 3.9812 +num rollout transitions: 50000, reward mean: 3.9883 +num rollout transitions: 50000, reward mean: 3.9782 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9896 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 3.9963 +num rollout transitions: 50000, reward mean: 4.0022 +---------------------------------------------------------------------------------- +| alpha | 0.102 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.844 | +| loss/actor | -361 | +| loss/alpha | -0.00659 | +| loss/critic | 43.1 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9861 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 3.9748 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 3.9891 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0071 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0043 +num rollout transitions: 50000, reward mean: 3.9819 +num rollout transitions: 50000, reward mean: 3.9912 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9849 +num rollout transitions: 50000, reward mean: 3.9835 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 4.0082 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9891 +num rollout transitions: 50000, reward mean: 4.0123 +num rollout transitions: 50000, reward mean: 3.9877 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0127 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 3.9822 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 3.9899 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 4.0201 +num rollout transitions: 50000, reward mean: 4.0223 +num rollout transitions: 50000, reward mean: 4.0009 +num rollout transitions: 50000, reward mean: 4.0271 +num rollout transitions: 50000, reward mean: 4.0174 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 4.0087 +---------------------------------------------------------------------------------- +| alpha | 0.106 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.851 | +| loss/actor | -359 | +| loss/alpha | 0.00667 | +| loss/critic | 42.6 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0201 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0262 +num rollout transitions: 50000, reward mean: 4.0122 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 3.9929 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 3.9921 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0214 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0032 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 3.9829 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 3.9938 +num rollout transitions: 50000, reward mean: 4.0154 +num rollout transitions: 50000, reward mean: 4.0052 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0193 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 3.9833 +---------------------------------------------------------------------------------- +| alpha | 0.102 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.928 | +| loss/actor | -360 | +| loss/alpha | 0.014 | +| loss/critic | 45.2 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9864 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9932 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 4.0153 +num rollout transitions: 50000, reward mean: 3.9890 +num rollout transitions: 50000, reward mean: 3.9862 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0131 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 4.0209 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0162 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 4.0112 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 3.9959 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 4.0037 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 3.9935 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 4.0187 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0073 +---------------------------------------------------------------------------------- +| alpha | 0.104 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 2.35 | +| loss/actor | -366 | +| loss/alpha | -0.00707 | +| loss/critic | 44.4 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.27 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0113 +num rollout transitions: 50000, reward mean: 4.0119 +num rollout transitions: 50000, reward mean: 4.0192 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9980 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0132 +num rollout transitions: 50000, reward mean: 4.0103 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 3.9954 +num rollout transitions: 50000, reward mean: 3.9871 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 4.0106 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0136 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9870 +num rollout transitions: 50000, reward mean: 3.9962 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0007 +---------------------------------------------------------------------------------- +| alpha | 0.103 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 121 | +| eval/normalized_episode_reward_std | 0.316 | +| loss/actor | -366 | +| loss/alpha | -0.0119 | +| loss/critic | 45.4 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 4.0060 +num rollout transitions: 50000, reward mean: 3.9911 +num rollout transitions: 50000, reward mean: 4.0086 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0102 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 4.0145 +num rollout transitions: 50000, reward mean: 3.9967 +num rollout transitions: 50000, reward mean: 3.9861 +num rollout transitions: 50000, reward mean: 3.9841 +num rollout transitions: 50000, reward mean: 3.9808 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9976 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0045 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 4.0019 +num rollout transitions: 50000, reward mean: 4.0054 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0164 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 4.0137 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 3.9780 +num rollout transitions: 50000, reward mean: 4.0029 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9878 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 3.9992 +num rollout transitions: 50000, reward mean: 4.0017 +num rollout transitions: 50000, reward mean: 3.9869 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9901 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 3.9924 +num rollout transitions: 50000, reward mean: 4.0133 +---------------------------------------------------------------------------------- +| alpha | 0.105 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 119 | +| eval/normalized_episode_reward_std | 1.01 | +| loss/actor | -365 | +| loss/alpha | 0.0164 | +| loss/critic | 45.5 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 3.9999 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9879 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9848 +num rollout transitions: 50000, reward mean: 3.9934 +num rollout transitions: 50000, reward mean: 4.0020 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0146 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 3.9977 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 3.9874 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9842 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 3.9876 +num rollout transitions: 50000, reward mean: 4.0246 +num rollout transitions: 50000, reward mean: 4.0332 +num rollout transitions: 50000, reward mean: 4.0109 +num rollout transitions: 50000, reward mean: 4.0133 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0092 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 3.9843 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0070 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0048 +num rollout transitions: 50000, reward mean: 3.9948 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9975 +---------------------------------------------------------------------------------- +| alpha | 0.104 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.335 | +| loss/actor | -365 | +| loss/alpha | 0.0159 | +| loss/critic | 44.7 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.3 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0051 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0027 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9928 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 3.9836 +num rollout transitions: 50000, reward mean: 4.0116 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 3.9861 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9853 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9979 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 3.9975 +num rollout transitions: 50000, reward mean: 4.0073 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 3.9945 +num rollout transitions: 50000, reward mean: 3.9895 +num rollout transitions: 50000, reward mean: 3.9861 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0160 +num rollout transitions: 50000, reward mean: 4.0069 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 3.9844 +num rollout transitions: 50000, reward mean: 3.9940 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9906 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9955 +num rollout transitions: 50000, reward mean: 4.0115 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9918 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0021 +---------------------------------------------------------------------------------- +| alpha | 0.106 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 122 | +| eval/normalized_episode_reward_std | 0.761 | +| loss/actor | -359 | +| loss/alpha | -0.0052 | +| loss/critic | 46 | +| penalty/lcb_mean | 1.43 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0169 +num rollout transitions: 50000, reward mean: 4.0064 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0144 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 4.0142 +num rollout transitions: 50000, reward mean: 4.0188 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 4.0021 +num rollout transitions: 50000, reward mean: 3.9974 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9764 +num rollout transitions: 50000, reward mean: 3.9944 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 4.0080 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9968 +num rollout transitions: 50000, reward mean: 3.9889 +num rollout transitions: 50000, reward mean: 4.0161 +num rollout transitions: 50000, reward mean: 4.0024 +num rollout transitions: 50000, reward mean: 4.0072 +num rollout transitions: 50000, reward mean: 4.0041 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0182 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9878 +num rollout transitions: 50000, reward mean: 3.9909 +num rollout transitions: 50000, reward mean: 3.9931 +num rollout transitions: 50000, reward mean: 3.9949 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 3.9973 +num rollout transitions: 50000, reward mean: 4.0059 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0097 +num rollout transitions: 50000, reward mean: 4.0058 +num rollout transitions: 50000, reward mean: 4.0100 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9821 +---------------------------------------------------------------------------------- +| alpha | 0.099 | +| eval/episode_length | 996 | +| eval/episode_length_std | 40.5 | +| eval/normalized_episode_reward | 120 | +| eval/normalized_episode_reward_std | 5.41 | +| loss/actor | -367 | +| loss/alpha | -0.00719 | +| loss/critic | 46.3 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.37 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9802 +num rollout transitions: 50000, reward mean: 3.9900 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9993 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9996 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0079 +num rollout transitions: 50000, reward mean: 3.9939 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 3.9960 +num rollout transitions: 50000, reward mean: 3.9978 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0152 +num rollout transitions: 50000, reward mean: 3.9884 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0098 +num rollout transitions: 50000, reward mean: 3.9991 +num rollout transitions: 50000, reward mean: 4.0033 +num rollout transitions: 50000, reward mean: 4.0085 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0128 +num rollout transitions: 50000, reward mean: 4.0155 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0089 +num rollout transitions: 50000, reward mean: 4.0099 +num rollout transitions: 50000, reward mean: 4.0209 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 3.9983 +num rollout transitions: 50000, reward mean: 3.9915 +num rollout transitions: 50000, reward mean: 3.9873 +num rollout transitions: 50000, reward mean: 3.9872 +num rollout transitions: 50000, reward mean: 3.9995 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0167 +num rollout transitions: 50000, reward mean: 3.9943 +num rollout transitions: 50000, reward mean: 4.0105 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9877 +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 3.9913 +---------------------------------------------------------------------------------- +| alpha | 0.0981 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 121 | +| eval/normalized_episode_reward_std | 0.437 | +| loss/actor | -364 | +| loss/alpha | -0.0167 | +| loss/critic | 47.6 | +| penalty/lcb_mean | 1.43 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9922 +num rollout transitions: 50000, reward mean: 3.9946 +num rollout transitions: 50000, reward mean: 3.9863 +num rollout transitions: 50000, reward mean: 3.9933 +num rollout transitions: 50000, reward mean: 3.9855 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 3.9981 +num rollout transitions: 50000, reward mean: 3.9880 +num rollout transitions: 50000, reward mean: 3.9930 +num rollout transitions: 50000, reward mean: 3.9984 +num rollout transitions: 50000, reward mean: 4.0061 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0108 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0066 +num rollout transitions: 50000, reward mean: 4.0056 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0110 +num rollout transitions: 50000, reward mean: 3.9989 +num rollout transitions: 50000, reward mean: 4.0049 +num rollout transitions: 50000, reward mean: 4.0005 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 3.9956 +num rollout transitions: 50000, reward mean: 4.0001 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 3.9936 +num rollout transitions: 50000, reward mean: 3.9951 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 3.9894 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 4.0014 +num rollout transitions: 50000, reward mean: 3.9910 +num rollout transitions: 50000, reward mean: 3.9950 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0140 +num rollout transitions: 50000, reward mean: 4.0084 +num rollout transitions: 50000, reward mean: 4.0081 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 4.0087 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 3.9964 +---------------------------------------------------------------------------------- +| alpha | 0.102 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 122 | +| eval/normalized_episode_reward_std | 0.178 | +| loss/actor | -360 | +| loss/alpha | 0.00545 | +| loss/critic | 45.9 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.0007 +num rollout transitions: 50000, reward mean: 4.0023 +num rollout transitions: 50000, reward mean: 4.0088 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 3.9971 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 4.0093 +num rollout transitions: 50000, reward mean: 3.9942 +num rollout transitions: 50000, reward mean: 3.9916 +num rollout transitions: 50000, reward mean: 3.9913 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 3.9941 +num rollout transitions: 50000, reward mean: 4.0006 +num rollout transitions: 50000, reward mean: 4.0044 +num rollout transitions: 50000, reward mean: 4.0047 +num rollout transitions: 50000, reward mean: 4.0121 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0008 +num rollout transitions: 50000, reward mean: 4.0090 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9953 +num rollout transitions: 50000, reward mean: 4.0036 +num rollout transitions: 50000, reward mean: 3.9920 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9982 +num rollout transitions: 50000, reward mean: 3.9919 +num rollout transitions: 50000, reward mean: 4.0095 +num rollout transitions: 50000, reward mean: 4.0094 +num rollout transitions: 50000, reward mean: 4.0018 +num rollout transitions: 50000, reward mean: 4.0065 +num rollout transitions: 50000, reward mean: 4.0030 +num rollout transitions: 50000, reward mean: 4.0046 +num rollout transitions: 50000, reward mean: 4.0107 +num rollout transitions: 50000, reward mean: 4.0077 +num rollout transitions: 50000, reward mean: 3.9923 +num rollout transitions: 50000, reward mean: 4.0035 +num rollout transitions: 50000, reward mean: 3.9958 +num rollout transitions: 50000, reward mean: 4.0118 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 3.9947 +num rollout transitions: 50000, reward mean: 4.0129 +num rollout transitions: 50000, reward mean: 4.0002 +num rollout transitions: 50000, reward mean: 4.0012 +num rollout transitions: 50000, reward mean: 4.0068 +num rollout transitions: 50000, reward mean: 3.9886 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0000 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 4.0021 +---------------------------------------------------------------------------------- +| alpha | 0.098 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 122 | +| eval/normalized_episode_reward_std | 0.164 | +| loss/actor | -361 | +| loss/alpha | -0.00554 | +| loss/critic | 48.1 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 3.9907 +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.0031 +num rollout transitions: 50000, reward mean: 4.0038 +num rollout transitions: 50000, reward mean: 4.0003 +num rollout transitions: 50000, reward mean: 3.9969 +num rollout transitions: 50000, reward mean: 4.0013 +num rollout transitions: 50000, reward mean: 4.0028 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0010 +num rollout transitions: 50000, reward mean: 3.9961 +num rollout transitions: 50000, reward mean: 4.0062 +num rollout transitions: 50000, reward mean: 3.9957 +num rollout transitions: 50000, reward mean: 4.0004 +num rollout transitions: 50000, reward mean: 3.9885 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 4.0063 +num rollout transitions: 50000, reward mean: 4.0078 +num rollout transitions: 50000, reward mean: 4.0039 +num rollout transitions: 50000, reward mean: 4.0034 +num rollout transitions: 50000, reward mean: 4.0011 +num rollout transitions: 50000, reward mean: 4.0015 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 3.9952 +num rollout transitions: 50000, reward mean: 3.9987 +num rollout transitions: 50000, reward mean: 3.9997 +num rollout transitions: 50000, reward mean: 3.9990 +num rollout transitions: 50000, reward mean: 3.9887 +num rollout transitions: 50000, reward mean: 4.0111 +num rollout transitions: 50000, reward mean: 3.9964 +num rollout transitions: 50000, reward mean: 4.0075 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 4.0067 +num rollout transitions: 50000, reward mean: 4.0053 +num rollout transitions: 50000, reward mean: 3.9927 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0026 +num rollout transitions: 50000, reward mean: 4.0022 +num rollout transitions: 50000, reward mean: 3.9972 +num rollout transitions: 50000, reward mean: 3.9986 +num rollout transitions: 50000, reward mean: 4.0091 +num rollout transitions: 50000, reward mean: 3.9988 +num rollout transitions: 50000, reward mean: 4.0025 +num rollout transitions: 50000, reward mean: 4.0016 +num rollout transitions: 50000, reward mean: 3.9965 +num rollout transitions: 50000, reward mean: 4.0074 +num rollout transitions: 50000, reward mean: 4.0040 +num rollout transitions: 50000, reward mean: 4.0057 +num rollout transitions: 50000, reward mean: 3.9974 +---------------------------------------------------------------------------------- +| alpha | 0.0924 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 122 | +| eval/normalized_episode_reward_std | 0.318 | +| loss/actor | -361 | +| loss/alpha | 0.0387 | +| loss/critic | 45.2 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 1 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 120864.01s diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_456×tamp_26-0319-224114/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_456×tamp_26-0319-224114/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_456×tamp_26-0319-224114/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_456×tamp_26-0319-224114/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..3709d65f34421de42619185c68004e81ba9e93ca --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_456×tamp_26-0319-224114/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_diffusion.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 456, "device": "cuda:7", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_456×tamp_26-0112-030559/model", "reward_penalty_coef": 0.5, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/diffusion/checkpoint.pt", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "density_model": "diffusion", "data_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "penalty_type": "tanh", "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "diffusion"} diff --git a/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_456×tamp_26-0319-224114/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_456×tamp_26-0319-224114/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..dcf8b789a3e0ed25e925e3b5c8a5d6d4a3fcdd71 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/diffusion/seed_456×tamp_26-0319-224114/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.2553658136335001,213.34,85.45352187007859,10.836652153151249,5.789437312531804,-262.1381098307619,-1.6760147689224976,17.515108924390393,0.9026711702264908,1.1458404022523454,1.0,0.0,50000.0,4.0246295610616185,50000 +0.13225570298656686,131.91,93.35331756290186,3.6514703933595745,7.8617526124766055,-383.81583899658307,0.00022894478127349858,25.994250643749425,1.0567934375750758,1.5667164217519867,1.0,0.0,50000.0,4.003933436455261,100000 +0.1244152981106939,356.83,342.03175451995685,28.825563107395286,35.29479693275081,-391.69425093505714,-0.00900921213779603,25.05673027193055,1.0359681094348452,1.5498862945628364,1.0,0.0,50000.0,4.01307171461761,150000 +0.12727421524718335,213.3,130.089776692867,8.755179581278465,12.141730318906195,-388.18556571044934,0.013490224524475835,23.866633532419076,1.040570680717222,1.4507447761487964,1.0,0.0,50000.0,4.014599867682506,200000 +0.1331460579907897,220.76,241.72567592210802,15.598740324177,25.554378521755268,-374.5069481115668,0.00203466972634189,27.187578019409404,1.094145900868181,1.5251779661095144,1.0,0.0,50000.0,4.013536770302393,250000 +0.14049166617661374,602.72,408.67691591280266,55.485664722030705,44.14660130247577,-362.407778751836,-0.0018722127351165155,32.53658167976354,1.195259098563205,1.679775658146147,1.0,0.0,50000.0,4.01011306150335,300000 +0.13903249921977354,935.27,130.7703219388864,94.58540748756778,14.288685449823477,-357.3205105511493,-0.0030418247254938332,32.52501562267285,1.197557824088349,1.682339270596512,1.0,0.0,50000.0,4.004369308678365,350000 +0.13836604527503496,957.55,110.42376329395769,95.47471637457213,11.55738676487327,-358.48459486511484,0.0055660865835100515,33.48706586250277,1.193286537349252,1.7093258497285944,1.0,0.0,50000.0,4.002919225651656,400000 +0.13175356430560325,871.8,202.7590688477337,88.29033197318569,21.43475488640893,-360.4318855999801,-0.012002725830599638,34.08779840217553,1.1979680596137212,1.78078229809881,1.0,0.0,50000.0,4.003150745953642,450000 +0.12869892644047473,1000.0,0.0,105.4787966487122,2.150012775410649,-363.11621576293817,-0.005677014022581102,34.48657260432262,1.1920342094993568,1.830613628103726,1.0,0.0,50000.0,4.002563771005679,500000 +0.12445530277803557,938.55,121.42531655301542,97.79557620335284,14.223129799849746,-361.23819082336195,0.009566868717335232,34.053208493403396,1.1959775229692564,1.8247799315857849,1.0,0.0,50000.0,4.007378965633276,550000 +0.12882867706522358,982.93,66.046688789068,104.07798895044755,7.859377967085905,-362.5287406329367,-0.004932803852260045,37.6744556704902,1.2487125076914085,1.933332591583745,1.0,0.0,50000.0,4.000899082296527,600000 +0.12133945448532596,923.96,177.80460736437627,93.22656341538953,20.251961263643043,-364.47087170287915,-0.0019681011549010172,35.69393010187116,1.222082433050872,1.8935273389089085,1.0,0.0,50000.0,4.002398946537422,650000 +0.12317324328407574,963.75,91.03311210762818,98.14785433964364,10.552161004244997,-364.69853014281733,-0.0013157299340889144,38.277183175125074,1.2710588313996805,1.968998111095426,1.0,0.0,50000.0,4.000124759887423,700000 +0.12660117273807642,996.66,30.454956903597807,104.68695703421255,4.521756471723666,-363.5658560028079,0.009270019718743796,38.26579048551578,1.2638520929110002,1.9726042095017453,1.0,0.0,50000.0,3.995684348600029,750000 +0.12450735474094794,1000.0,0.0,105.5953787167113,1.1849175551125826,-361.2753722052009,-0.007707061821855657,38.15969769275659,1.263753811259269,1.9941974624931798,1.0,0.0,50000.0,3.998898471211289,800000 +0.12304781988784659,948.5,110.3831055913902,99.47540147584706,12.830425288075977,-364.57333188110005,-0.0076303198277207105,38.14217859392172,1.2616558093988801,2.0103221983909862,1.0,0.0,50000.0,3.9979835250396856,850000 +0.12042132622376103,1000.0,0.0,104.80345049452512,0.9232643577219143,-366.3926175354029,0.005256377189718322,39.00981483249646,1.272498369154929,2.045253836535171,1.0,0.0,50000.0,4.0000943254167565,900000 +0.11930252486437556,1000.0,0.0,113.1681741066677,1.3529718514649332,-364.80510215210205,0.003886750590279686,39.60386731964049,1.2751722973585153,2.0696923989105254,1.0,0.0,50000.0,4.004430673115872,950000 +0.11866037332579574,994.44,31.864500623734873,108.4234392078298,3.88085473451272,-366.3129516656464,-0.001364726562798034,38.48869056510887,1.2587556154215227,2.0127723715496066,1.0,0.0,50000.0,4.00156571110388,1000000 +0.12090066632613501,984.14,77.84638976856922,108.69599638677374,9.145546585886365,-364.04188069030045,0.0016769585960358213,40.02523115320195,1.2743240040898471,2.1022187823415055,1.0,0.0,50000.0,4.000672738152743,1050000 +0.11748789438351964,1000.0,0.0,107.43152700958237,1.075045899850509,-362.7634710986249,-0.008261860500872065,38.020794530525045,1.2589779070317753,2.015316349046259,1.0,0.0,50000.0,4.000722481755138,1100000 +0.11855962048754083,1000.0,0.0,110.16553044263885,1.6510431890608797,-363.3117575079323,0.004085082106217658,40.67374458452191,1.3216379606676294,2.084801079646381,1.0,0.0,50000.0,3.996568383319438,1150000 +0.1241841416703166,988.04,65.25563883680857,108.20296717625996,7.954756434443183,-360.91829438049217,0.008740396332964471,40.347903830165656,1.3301017166352327,2.077566479867718,1.0,0.0,50000.0,3.9987641634767286,1200000 +0.11607089617937827,993.27,45.325678152676325,110.85431837736965,5.880415243085202,-364.1161291668685,-0.012157881575152244,38.493221775436545,1.2738449699771355,2.029670125216254,1.0,0.0,50000.0,3.9963553372185827,1250000 +0.11599816232025553,997.01,29.750124369487953,114.60803192667757,3.9825930466174726,-362.737237722778,0.010991426539085879,38.34572555448526,1.285354634106139,2.054962743265619,1.0,0.0,50000.0,4.001523505388594,1300000 +0.11685223326295599,1000.0,0.0,114.64024138975309,2.9694818583664433,-362.2895743536342,-0.002775514878369748,39.41793151084898,1.2938490564203444,2.1129807722222735,1.0,0.0,50000.0,4.000989997410037,1350000 +0.1159499410434061,958.39,123.88784403645097,103.26186879182136,14.615793999853565,-361.2755137402316,0.0069211850215492665,39.6152085315897,1.3076794428444178,2.1137407743132175,1.0,0.0,50000.0,4.0013160934677465,1400000 +0.11986180950969542,1000.0,0.0,108.41964126487372,2.0937516698382796,-359.0429083502154,0.006779924673102835,40.42247987232151,1.3315991015148203,2.1026909313595405,1.0,0.0,50000.0,4.0020844957896,1450000 +0.11816633291974758,994.26,51.5180783803123,109.02717910435187,6.505265040416917,-359.0803190197785,-0.012919372180774824,40.701932094173316,1.3227094557201693,2.118593434278906,1.0,0.0,50000.0,4.001696519620346,1500000 +0.12269127929076486,935.58,146.37111600312406,97.33215860266675,16.1992503156615,-358.6851351507571,0.009862861130796149,44.184596162700736,1.37996972073554,2.195351347212801,1.0,0.0,50000.0,4.000620969150854,1550000 +0.12381108089134085,1000.0,0.0,116.94715652299466,1.1102762630859107,-358.6203343341087,-0.009636649190336405,44.194670819931076,1.3923823174917758,2.2128672956407103,1.0,0.0,50000.0,3.997116451779222,1600000 +0.12363316514671041,1000.0,0.0,110.8770579178215,0.7066916146059747,-360.79694925293046,0.006148805913664433,43.81871369424897,1.361021070857045,2.249754822005066,1.0,0.0,50000.0,4.004303001621746,1650000 +0.11906146877974194,1000.0,0.0,116.9143722605661,0.505152169991207,-358.3419304846179,-0.002416115830857388,42.79576930917749,1.342226336377843,2.2045843422186597,1.0,0.0,50000.0,4.000394704765381,1700000 +0.11959301894605218,1000.0,0.0,110.61226236508566,1.0589775398474033,-359.21497645385983,-0.00020213801726695163,43.73559278907819,1.3565182521271473,2.2253058465218527,1.0,0.0,50000.0,3.999736388620831,1750000 +0.11714726765140855,993.26,40.407331018021964,113.20870156756388,5.2085463567374,-356.21685776489284,-0.0026036050000041917,41.715407807674666,1.3382556811642643,2.175769233256568,1.0,0.0,50000.0,4.000845215658377,1800000 +0.11490281362444199,994.51,54.62481029715346,112.04631946309087,7.090544847903167,-360.82082049927465,0.0005828511572629608,40.90856382802969,1.307326722278586,2.1587128133201428,1.0,0.0,50000.0,4.010782535556352,1850000 +0.11284015809535873,1000.0,0.0,110.2624210876154,1.0232158967652627,-360.0522564691126,-0.001639198982342964,43.236333236255504,1.3200201134574465,2.2407274284041283,1.0,0.0,50000.0,4.001855039439878,1900000 +0.11296520399674671,1000.0,0.0,115.43110865653196,2.3980044423514157,-359.34948292968977,0.009458009369336242,41.38100824464866,1.3326539751434392,2.1991950276327428,1.0,0.0,50000.0,4.003639594948329,1950000 +0.11421563003852932,995.86,27.19449208939192,120.1028710463157,3.9743635973841576,-358.58438118102725,-0.0016792802120742323,43.51003249860695,1.334956593419324,2.200023343330579,1.0,0.0,50000.0,4.005326004627931,2000000 +0.11938240830108551,1000.0,0.0,115.93913016641801,0.8628579768426844,-357.50986110473394,-0.008690745934955684,43.364926205463405,1.3555033354985726,2.1988816235232598,1.0,0.0,50000.0,4.0039375505117425,2050000 +0.10902919512793384,968.49,100.67040230375561,107.53173872878934,11.940087481139049,-361.703130438227,-0.0005951299800723581,43.735877128429514,1.3477722207784404,2.2073936100196687,1.0,0.0,50000.0,4.004886467765881,2100000 +0.11746560625478585,1000.0,0.0,118.55088129239002,0.5279023822585819,-357.82416324768553,0.008833638616614072,46.57799502708477,1.402195908255577,2.3008000000345423,1.0,0.0,50000.0,4.003372463711355,2150000 +0.11572342074632615,982.15,82.98871911290114,115.70922516864938,10.892406593548806,-357.88951137878155,0.006069162555784117,44.25662681236234,1.375461717541229,2.2557010994231743,1.0,0.0,50000.0,4.005159292716457,2200000 +0.11534092853203569,955.19,122.06168071921671,106.45777324039096,14.46916555356216,-360.7452335522481,-0.005521236979887018,47.315983747692876,1.3941189448154163,2.3373532949662343,1.0,0.0,50000.0,4.0025237940784315,2250000 +0.11558126486286399,1000.0,0.0,117.98171163475466,1.1181734599488218,-358.43666761413186,-0.007191081157661632,46.378575260486436,1.4052448690223576,2.326503875552372,1.0,0.0,50000.0,4.002603388530028,2300000 +0.11145214978933325,1000.0,0.0,118.55797295332904,0.8316481769768782,-359.45878108581417,-0.01956092443984013,43.82391653120052,1.3593333022928162,2.2398380184411883,1.0,0.0,50000.0,4.000984358212362,2350000 +0.10203307731240978,1000.0,0.0,116.12278007319993,0.8435251991333078,-361.41566930968673,-0.006586252990700297,43.14294662830245,1.3499206274628726,2.240120150139386,1.0,0.0,50000.0,3.9981329326196673,2400000 +0.10558660709664394,1000.0,0.0,116.05547399442426,0.8513506819461497,-359.21644316711354,0.006669631282500867,42.57562793580907,1.3593233826744733,2.2372231042789985,1.0,0.0,50000.0,3.9993053732153885,2450000 +0.10246429434269724,1000.0,0.0,116.78464850026118,0.9283676327273163,-360.3170580676273,0.014025460636839383,45.246640207920656,1.3826805600452345,2.2841706864106928,1.0,0.0,50000.0,4.00228134549917,2500000 +0.10364287561670042,1000.0,0.0,117.83704863970314,2.353025621499142,-366.17300035827407,-0.007067393711022938,44.36256688871436,1.3647568035089677,2.267720844463137,1.0,0.0,50000.0,4.001859835017836,2550000 +0.10297108450397806,1000.0,0.0,121.30529099937408,0.31611315185505484,-365.8824483721916,-0.01194609496455637,45.4357603264615,1.3954708277249457,2.3328723291611797,1.0,0.0,50000.0,4.002921937667262,2600000 +0.10517658187657722,1000.0,0.0,119.47692918676492,1.0094120812387342,-365.06727575378466,0.01641010856546557,45.468820202770104,1.3908113813221306,2.332919401013851,1.0,0.0,50000.0,4.00083284399233,2650000 +0.10436446571797092,1000.0,0.0,117.29125729766454,0.3353731089281553,-365.3827030596923,0.015947532604001467,44.69512223140799,1.4118120820844156,2.30309964722513,1.0,0.0,50000.0,4.001161173888183,2700000 +0.10561971209526011,1000.0,0.0,121.6921445842152,0.761212963473292,-359.4966991662595,-0.0052006159218404725,46.00723925745016,1.4276224862551643,2.330930557001858,1.0,0.0,50000.0,3.997872470553279,2750000 +0.09896056702092164,995.93,40.49598869023943,119.50803737702739,5.409749488514856,-367.03865041198844,-0.007193946697227467,46.3180597191618,1.4166895350909459,2.373626206233478,1.0,0.0,50000.0,4.001649618824731,2800000 +0.09812343762993822,1000.0,0.0,121.49558451275801,0.43681925182811937,-364.4406291027768,-0.0167120004246381,47.58563589038898,1.425321735332004,2.3814909381616545,1.0,0.0,50000.0,4.001443091266989,2850000 +0.10215486692354066,1000.0,0.0,121.67895403182418,0.1780390820248533,-360.0645223187292,0.005449310096204787,45.917401694488944,1.418543178334224,2.341468149044522,1.0,0.0,50000.0,4.001311959584106,2900000 +0.09800074114158638,1000.0,0.0,122.22058398417627,0.16380777243076614,-360.8367860443149,-0.00553649482380588,48.0643442004009,1.4570112673866673,2.395924437878102,1.0,0.0,50000.0,4.001543977747701,2950000 +0.09235550266161445,1000.0,0.0,121.946065441735,0.31829897071352353,-360.53255439818935,0.038700756597780425,45.17324142825987,1.4210318074798673,2.325565914064627,1.0,0.0,50000.0,4.000846462472199,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_123×tamp_26-0505-202558/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/kde/seed_123×tamp_26-0505-202558/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..8dc1443eb06372e1f33d327daee83437d44b4b6c --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/kde/seed_123×tamp_26-0505-202558/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.5675 +num rollout transitions: 50000, reward mean: 4.5480 +num rollout transitions: 50000, reward mean: 4.5360 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5290 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5319 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5305 +num rollout transitions: 50000, reward mean: 4.5409 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5353 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4694 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5316 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4762 +---------------------------------------------------------------------------------- +| alpha | 0.255 | +| eval/episode_length | 34.3 | +| eval/episode_length_std | 0.57 | +| eval/normalized_episode_reward | 0.0794 | +| eval/normalized_episode_reward_std | -0.0205 | +| loss/actor | -290 | +| loss/alpha | -2.06 | +| loss/critic | 22.9 | +| penalty/lcb_mean | 1.02 | +| penalty/lcb_std | 1.36 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.4716 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5339 +num rollout transitions: 50000, reward mean: 4.5304 +num rollout transitions: 50000, reward mean: 4.5241 +num rollout transitions: 50000, reward mean: 4.5319 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5330 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.4740 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5304 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.5038 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 120 | +| eval/episode_length_std | 18.5 | +| eval/normalized_episode_reward | 5.36 | +| eval/normalized_episode_reward_std | 1.77 | +| loss/actor | -451 | +| loss/alpha | 0.0247 | +| loss/critic | 33.3 | +| penalty/lcb_mean | 1.21 | +| penalty/lcb_std | 1.87 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5222 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5032 +---------------------------------------------------------------------------------- +| alpha | 0.152 | +| eval/episode_length | 274 | +| eval/episode_length_std | 132 | +| eval/normalized_episode_reward | 8.14 | +| eval/normalized_episode_reward_std | 14.4 | +| loss/actor | -454 | +| loss/alpha | -0.00445 | +| loss/critic | 34.2 | +| penalty/lcb_mean | 1.21 | +| penalty/lcb_std | 1.8 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5276 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5319 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5265 +num rollout transitions: 50000, reward mean: 4.5044 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 124 | +| eval/episode_length_std | 5.89 | +| eval/normalized_episode_reward | 7.18 | +| eval/normalized_episode_reward_std | 0.59 | +| loss/actor | -439 | +| loss/alpha | 0.0083 | +| loss/critic | 35.2 | +| penalty/lcb_mean | 1.24 | +| penalty/lcb_std | 1.75 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5283 +num rollout transitions: 50000, reward mean: 4.5205 +num rollout transitions: 50000, reward mean: 4.5384 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5291 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5223 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5082 +---------------------------------------------------------------------------------- +| alpha | 0.149 | +| eval/episode_length | 759 | +| eval/episode_length_std | 257 | +| eval/normalized_episode_reward | 64.9 | +| eval/normalized_episode_reward_std | 25.1 | +| loss/actor | -425 | +| loss/alpha | -0.00641 | +| loss/critic | 36 | +| penalty/lcb_mean | 1.24 | +| penalty/lcb_std | 1.78 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5324 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5391 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5327 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5329 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5023 +---------------------------------------------------------------------------------- +| alpha | 0.157 | +| eval/episode_length | 813 | +| eval/episode_length_std | 229 | +| eval/normalized_episode_reward | 80.2 | +| eval/normalized_episode_reward_std | 24.2 | +| loss/actor | -411 | +| loss/alpha | -0.00115 | +| loss/critic | 38.8 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5282 +num rollout transitions: 50000, reward mean: 4.5205 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5271 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5205 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4969 +---------------------------------------------------------------------------------- +| alpha | 0.153 | +| eval/episode_length | 221 | +| eval/episode_length_std | 184 | +| eval/normalized_episode_reward | 16.7 | +| eval/normalized_episode_reward_std | 20.3 | +| loss/actor | -406 | +| loss/alpha | 0.00273 | +| loss/critic | 40 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.94 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5217 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5124 +---------------------------------------------------------------------------------- +| alpha | 0.154 | +| eval/episode_length | 152 | +| eval/episode_length_std | 6.47 | +| eval/normalized_episode_reward | 7.84 | +| eval/normalized_episode_reward_std | 0.2 | +| loss/actor | -406 | +| loss/alpha | -0.00326 | +| loss/critic | 40.3 | +| penalty/lcb_mean | 1.28 | +| penalty/lcb_std | 1.95 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5270 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.4830 +---------------------------------------------------------------------------------- +| alpha | 0.149 | +| eval/episode_length | 822 | +| eval/episode_length_std | 218 | +| eval/normalized_episode_reward | 85.5 | +| eval/normalized_episode_reward_std | 25 | +| loss/actor | -409 | +| loss/alpha | -0.00108 | +| loss/critic | 41.5 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 1.99 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4820 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5217 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5173 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 0.661 | +| loss/actor | -410 | +| loss/alpha | -0.00781 | +| loss/critic | 40.6 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 1.98 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5267 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4811 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5223 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5211 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 966 | +| eval/episode_length_std | 120 | +| eval/normalized_episode_reward | 99.2 | +| eval/normalized_episode_reward_std | 13 | +| loss/actor | -413 | +| loss/alpha | 0.00265 | +| loss/critic | 41.7 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 2.04 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.5223 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5223 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5213 +----------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 909 | +| eval/episode_length_std | 188 | +| eval/normalized_episode_reward | 91.2 | +| eval/normalized_episode_reward_std | 20.4 | +| loss/actor | -409 | +| loss/alpha | -0.000482 | +| loss/critic | 44.2 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.11 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 600000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4769 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.4826 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4805 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5040 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 980 | +| eval/episode_length_std | 77.2 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 8.8 | +| loss/actor | -411 | +| loss/alpha | -0.0121 | +| loss/critic | 47 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.18 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4825 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4807 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4804 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5242 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.4847 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4900 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 138 | +| eval/episode_length_std | 3.45 | +| eval/normalized_episode_reward | 7.69 | +| eval/normalized_episode_reward_std | 0.309 | +| loss/actor | -411 | +| loss/alpha | 0.013 | +| loss/critic | 49 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4817 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5265 +num rollout transitions: 50000, reward mean: 4.4767 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4816 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4977 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.89 | +| loss/actor | -411 | +| loss/alpha | -0.00593 | +| loss/critic | 49.7 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.25 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4833 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4664 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.4797 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5049 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 951 | +| eval/episode_length_std | 125 | +| eval/normalized_episode_reward | 99.9 | +| eval/normalized_episode_reward_std | 14.3 | +| loss/actor | -408 | +| loss/alpha | 0.000453 | +| loss/critic | 48.5 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4812 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5268 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4780 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4846 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.504 | +| loss/actor | -409 | +| loss/alpha | -0.00367 | +| loss/critic | 50.2 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.3 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4779 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4726 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4825 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4731 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4908 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 989 | +| eval/episode_length_std | 71.1 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 8.52 | +| loss/actor | -412 | +| loss/alpha | -0.00515 | +| loss/critic | 47.8 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.23 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4798 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.4795 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5039 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 1.2 | +| loss/actor | -413 | +| loss/alpha | -0.00462 | +| loss/critic | 51 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.4857 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4807 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.628 | +| loss/actor | -408 | +| loss/alpha | 0.0123 | +| loss/critic | 48.9 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.25 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.4670 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.4725 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4795 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 886 | +| eval/episode_length_std | 170 | +| eval/normalized_episode_reward | 89.6 | +| eval/normalized_episode_reward_std | 19.1 | +| loss/actor | -405 | +| loss/alpha | 8.48e-05 | +| loss/critic | 50.7 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4822 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4843 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5264 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4928 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 999 | +| eval/episode_length_std | 9.05 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 1.57 | +| loss/actor | -405 | +| loss/alpha | -0.0126 | +| loss/critic | 51.5 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4843 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4826 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5226 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.4998 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 946 | +| eval/episode_length_std | 120 | +| eval/normalized_episode_reward | 98.5 | +| eval/normalized_episode_reward_std | 13.9 | +| loss/actor | -407 | +| loss/alpha | 0.00324 | +| loss/critic | 50 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4657 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4797 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4820 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4828 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.4873 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 837 | +| eval/episode_length_std | 238 | +| eval/normalized_episode_reward | 84.9 | +| eval/normalized_episode_reward_std | 26.9 | +| loss/actor | -411 | +| loss/alpha | -0.00234 | +| loss/critic | 50.3 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4736 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4798 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4746 +num rollout transitions: 50000, reward mean: 4.4823 +num rollout transitions: 50000, reward mean: 4.4755 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.4776 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4757 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5053 +----------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 558 | +| eval/episode_length_std | 257 | +| eval/normalized_episode_reward | 56.5 | +| eval/normalized_episode_reward_std | 28.9 | +| loss/actor | -408 | +| loss/alpha | -0.000889 | +| loss/critic | 51.7 | +| penalty/lcb_mean | 1.43 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1250000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4825 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.5277 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5152 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.615 | +| loss/actor | -410 | +| loss/alpha | -0.00135 | +| loss/critic | 51 | +| penalty/lcb_mean | 1.43 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.5271 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4817 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5039 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.5 | +| loss/actor | -410 | +| loss/alpha | 0.0106 | +| loss/critic | 51.2 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5035 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 993 | +| eval/episode_length_std | 51.1 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 5.9 | +| loss/actor | -406 | +| loss/alpha | -0.00449 | +| loss/critic | 52.9 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4669 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4820 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 0.482 | +| loss/actor | -409 | +| loss/alpha | -0.0114 | +| loss/critic | 51.8 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4735 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5234 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4985 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 0.554 | +| loss/actor | -410 | +| loss/alpha | 0.0161 | +| loss/critic | 50.7 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4807 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.4825 +num rollout transitions: 50000, reward mean: 4.5055 +----------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 0.449 | +| loss/actor | -404 | +| loss/alpha | -0.000996 | +| loss/critic | 54 | +| penalty/lcb_mean | 1.45 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1550000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5273 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5333 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4986 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 0.209 | +| loss/actor | -402 | +| loss/alpha | -0.0145 | +| loss/critic | 53.5 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4831 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5265 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5071 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.971 | +| loss/actor | -408 | +| loss/alpha | 0.00884 | +| loss/critic | 54.9 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5404 +num rollout transitions: 50000, reward mean: 4.5223 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4950 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 973 | +| eval/episode_length_std | 131 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 15.9 | +| loss/actor | -410 | +| loss/alpha | 0.000652 | +| loss/critic | 53.1 | +| penalty/lcb_mean | 1.45 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5307 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5009 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.163 | +| loss/actor | -409 | +| loss/alpha | 0.00219 | +| loss/critic | 55.8 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.43 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5067 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.251 | +| loss/actor | -410 | +| loss/alpha | -0.00196 | +| loss/critic | 55.9 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4755 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5124 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.45 | +| loss/actor | -412 | +| loss/alpha | -0.00811 | +| loss/critic | 54.9 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.43 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4753 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4823 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5226 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4802 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4840 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5010 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 994 | +| eval/episode_length_std | 56.8 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 7.11 | +| loss/actor | -410 | +| loss/alpha | 0.0104 | +| loss/critic | 55.9 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5234 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4802 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 742 | +| eval/episode_length_std | 262 | +| eval/normalized_episode_reward | 82 | +| eval/normalized_episode_reward_std | 32.1 | +| loss/actor | -409 | +| loss/alpha | -0.00693 | +| loss/critic | 53.3 | +| penalty/lcb_mean | 1.45 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5275 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5244 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5280 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5219 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4992 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.23 | +| loss/actor | -411 | +| loss/alpha | 0.00298 | +| loss/critic | 54.1 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5336 +num rollout transitions: 50000, reward mean: 4.5253 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5331 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5258 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5245 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5135 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.22 | +| loss/actor | -408 | +| loss/alpha | 0.00286 | +| loss/critic | 58.1 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.48 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5005 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.477 | +| loss/actor | -409 | +| loss/alpha | 0.00501 | +| loss/critic | 53.8 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5279 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4771 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4919 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.205 | +| loss/actor | -410 | +| loss/alpha | -0.0126 | +| loss/critic | 53.8 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.43 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5205 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5274 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5242 +num rollout transitions: 50000, reward mean: 4.5271 +---------------------------------------------------------------------------------- +| alpha | 0.118 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.886 | +| loss/actor | -412 | +| loss/alpha | -0.00304 | +| loss/critic | 55.4 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5256 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4773 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4825 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5284 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4894 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.282 | +| loss/actor | -409 | +| loss/alpha | -0.00868 | +| loss/critic | 58.2 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4797 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4912 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.424 | +| loss/actor | -408 | +| loss/alpha | 0.00417 | +| loss/critic | 58.5 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.51 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5234 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5253 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4752 +num rollout transitions: 50000, reward mean: 4.4707 +num rollout transitions: 50000, reward mean: 4.4807 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5283 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4979 +---------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.848 | +| loss/actor | -406 | +| loss/alpha | 0.0111 | +| loss/critic | 57.6 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5071 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.271 | +| loss/actor | -402 | +| loss/alpha | 0.000676 | +| loss/critic | 64.6 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4905 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.864 | +| loss/actor | -403 | +| loss/alpha | -0.0101 | +| loss/critic | 62 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4774 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4937 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.752 | +| loss/actor | -406 | +| loss/alpha | 0.00557 | +| loss/critic | 61.4 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.6 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4825 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4976 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.661 | +| loss/actor | -405 | +| loss/alpha | -0.00358 | +| loss/critic | 61.2 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4949 +---------------------------------------------------------------------------------- +| alpha | 0.118 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.383 | +| loss/actor | -406 | +| loss/alpha | 0.00845 | +| loss/critic | 59.8 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4967 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.247 | +| loss/actor | -408 | +| loss/alpha | -0.00395 | +| loss/critic | 57.7 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4843 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4792 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5084 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.302 | +| loss/actor | -403 | +| loss/alpha | 0.0147 | +| loss/critic | 60.3 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.57 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4826 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5069 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.379 | +| loss/actor | -401 | +| loss/alpha | -0.0102 | +| loss/critic | 62.1 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.6 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4960 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 0.356 | +| loss/actor | -401 | +| loss/alpha | 0.00185 | +| loss/critic | 65.3 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.68 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4796 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4975 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.457 | +| loss/actor | -403 | +| loss/alpha | -0.0148 | +| loss/critic | 66.3 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.71 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5237 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 993 | +| eval/episode_length_std | 44.2 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 5.6 | +| loss/actor | -405 | +| loss/alpha | 0.017 | +| loss/critic | 66.2 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.71 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5049 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 998 | +| eval/episode_length_std | 15.6 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 2.01 | +| loss/actor | -402 | +| loss/alpha | 0.0111 | +| loss/critic | 64.8 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.7 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4997 +---------------------------------------------------------------------------------- +| alpha | 0.104 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.361 | +| loss/actor | -394 | +| loss/alpha | -0.0243 | +| loss/critic | 67.6 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.74 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 49615.72s diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_123×tamp_26-0505-202558/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/kde/seed_123×tamp_26-0505-202558/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_123×tamp_26-0505-202558/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/kde/seed_123×tamp_26-0505-202558/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..587fc29a8b9b00d87733ec4656f28eb4725baea4 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/kde/seed_123×tamp_26-0505-202558/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 123, "device": "cuda:5", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_123×tamp_26-0112-030558/model", "reward_penalty_coef": 0.3, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/kde", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "kde", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "kde"} diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_123×tamp_26-0505-202558/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/kde/seed_123×tamp_26-0505-202558/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..824f90028b73d4578327cd7a2609f17095d63ecb --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/kde/seed_123×tamp_26-0505-202558/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.2547949835097794,34.34,0.5695612346359257,0.07941838366875442,-0.020532631220816725,-290.49500504816643,-2.0558614829178103,22.876054335658228,1.0241636829726464,1.356810376052423,0.0,0.0,50000.0,4.508995422387819,50000 +0.13494160804823235,120.03,18.459390564154603,5.360843548658921,1.7664690453220795,-450.9135121411141,0.024680390909835623,33.3023476542856,1.2125116176033088,1.8656717903947746,0.0,0.0,50000.0,4.509430903486668,100000 +0.15205855684876426,273.82,131.95032247023877,8.136418742115485,14.357382109286768,-453.79887282958765,-0.004453645427934829,34.22353768232308,1.213978635556688,1.803023955084075,0.0,0.0,50000.0,4.507212415983638,150000 +0.1454487002930047,123.9,5.894913061275798,7.175863201674091,0.5895513110694143,-439.2310873925848,0.008298710858151405,35.209430638541896,1.2426973992800756,1.7524913017952244,0.0,0.0,50000.0,4.51167439939636,200000 +0.14934009483873784,758.77,256.9126254196161,64.86750347027078,25.120530420132493,-425.4187447955323,-0.006412655211873418,35.96573998205223,1.2420509185481137,1.7825282605564607,0.0,0.0,50000.0,4.511707568880102,250000 +0.15672630576372026,813.41,228.63604680802194,80.16482948507935,24.165750693833413,-410.5596369970622,-0.001153166715204497,38.81914317247382,1.26360560514333,1.8782440797817495,0.0,0.0,50000.0,4.510871473007439,300000 +0.1528643960970612,220.61,184.36718227493742,16.676350674619123,20.295957129818255,-405.9005148718301,0.0027333230672030814,40.03062690246551,1.2673629266786255,1.9429119264924495,0.0,0.0,50000.0,4.507209853701399,350000 +0.1540459327545773,152.17,6.466923534417273,7.837464883111646,0.20010397128438517,-405.7078336047348,-0.003263485361933747,40.33618769159252,1.277220643907767,1.9497214949238024,0.0,0.0,50000.0,4.50610351155246,400000 +0.14854064195275232,821.89,217.7622508608873,85.52210030713512,25.004859758960592,-408.9376776855438,-0.0010758158647827581,41.50644500024819,1.3048196735120128,1.988807697175739,0.0,0.0,50000.0,4.504079592134437,450000 +0.14529179633289768,1000.0,0.0,101.82191376363919,0.6607313630881338,-410.2829910516423,-0.007806521105133005,40.61070209522236,1.2929697626161445,1.977486113277688,0.0,0.0,50000.0,4.502265569080731,500000 +0.13279889104887782,965.86,119.74898913978356,99.18821772703262,12.996815657792174,-412.7672831414811,0.0026504267138614517,41.654935114441784,1.2881579781138752,2.0434072827803718,0.0,0.0,50000.0,4.503485452648428,550000 +0.13135037703886876,909.21,188.38287050578666,91.23591401036528,20.42833040324443,-409.4813255157499,-0.0004819422541185676,44.22174003917663,1.3374602444660337,2.1090955152093938,0.0,0.0,50000.0,4.506604710985084,600000 +0.1338408699128033,979.73,77.16785017090989,103.14809161826024,8.800165566234295,-410.8693758264132,-0.012148023413345112,46.9869694936562,1.3979952424287876,2.1779027230989665,0.0,0.0,50000.0,4.501640878334805,650000 +0.13936789891794388,137.9,3.448187929913333,7.688455421420246,0.3087220094467726,-410.51715178100494,0.012971217895373404,48.98352102390313,1.4177790585362844,2.2374677265334335,0.0,0.0,50000.0,4.49787680312176,700000 +0.14001309516698068,1000.0,0.0,111.24137598434392,0.8897140813223939,-411.22586616821314,-0.005930155657678984,49.72303252012185,1.413768317029478,2.246396863141098,0.0,0.0,50000.0,4.501678890000002,750000 +0.13367017117991847,950.71,125.395477988642,99.93943946033546,14.292559966994736,-407.58935521668,0.0004527554978803471,48.53646509120988,1.3789547246730192,2.24423492955088,0.0,0.0,50000.0,4.500855874238966,800000 +0.13119607234626815,1000.0,0.0,109.56031819147469,0.50377755578904,-408.8535864880378,-0.0036654009104146955,50.16548373744972,1.4067800636696774,2.296254518196614,0.0,0.0,50000.0,4.4997122552762585,850000 +0.12732408769399142,988.56,71.12359945897,110.23053596727455,8.522252688352538,-411.64082788085665,-0.005154991986565287,47.771110463447336,1.3742767801976403,2.226560092407474,0.0,0.0,50000.0,4.498122951226072,900000 +0.13454822590693868,1000.0,0.0,107.71330886732858,1.1951994732622775,-412.56595572021814,-0.004624391651525984,50.96390602687868,1.4187824236607756,2.2828754538715152,0.0,0.0,50000.0,4.503816060381727,950000 +0.1329061329418453,1000.0,0.0,111.43055963288016,0.6283987783425397,-408.14975401184296,0.012297814833447385,48.854355261001096,1.3894972366392548,2.2459991165888584,0.0,0.0,50000.0,4.500023716444701,1000000 +0.13466804491356224,885.51,169.93842973265345,89.55267652833072,19.063633977161732,-405.1289027642859,8.483955752097328e-05,50.72278399690606,1.4188833180224971,2.3239050651788817,0.0,0.0,50000.0,4.499084202542953,1050000 +0.1325007852055127,999.09,9.054385677670242,104.6408022950318,1.5710203750281666,-405.05890499756305,-0.012606782082430894,51.54025925550419,1.3990434376561727,2.3227708576237833,0.0,0.0,50000.0,4.504913405918644,1100000 +0.12764148666158165,946.29,120.0678387412716,98.54484654920151,13.931801322825555,-406.8276685766627,0.0032433511159569296,49.97235726596837,1.3942079838323673,2.314051734207853,0.0,0.0,50000.0,4.50461723518997,1150000 +0.12443165607437567,836.56,238.05651933942076,84.89495247931904,26.867920761419324,-411.05926236084156,-0.002340023654289512,50.26982113376672,1.4069527024054571,2.3418259924042513,0.0,0.0,50000.0,4.49660348621142,1200000 +0.12820488684847836,557.85,257.1479097717887,56.52102394233452,28.88082884742581,-408.060424134517,-0.0008885974261164806,51.65081123960508,1.4252065445685285,2.3417927490127473,0.0,0.0,50000.0,4.4969384490122595,1250000 +0.12899784868985495,1000.0,0.0,109.96073753096994,0.6151282689105168,-410.3139389306665,-0.001352357801608753,51.03088010009768,1.4286127735293046,2.337969925315423,0.0,0.0,50000.0,4.504812299225526,1300000 +0.12685785459608037,1000.0,0.0,111.34036383303834,0.49989329015438067,-409.9603603265352,0.010647197080142988,51.227447144203666,1.4362062185740527,2.3409627344417645,0.0,0.0,50000.0,4.505317795457468,1350000 +0.13471405367866052,992.74,51.11841546840042,104.4482689349066,5.900386291619119,-405.6328396350118,-0.004486505436189384,52.91641194286356,1.4702760292470665,2.34848608919384,0.0,0.0,50000.0,4.504409596483249,1400000 +0.12836743181035065,1000.0,0.0,108.28359896332654,0.48244849044122556,-408.7089697033706,-0.01136059780441224,51.75293170642846,1.4379670064175123,2.346717848111399,0.0,0.0,50000.0,4.501134321902985,1450000 +0.12560351707085993,1000.0,0.0,108.4211120989679,0.554057612465359,-410.01065328064294,0.01606284568537041,50.68911274105049,1.4211881534707491,2.3237952395117314,0.0,0.0,50000.0,4.503070620109077,1500000 +0.12694959209099282,1000.0,0.0,107.4331428047388,0.4487593826472726,-403.8354568072498,-0.0009963825992494256,53.988765341606,1.4453092457020544,2.388755494929551,0.0,0.0,50000.0,4.502078706058289,1550000 +0.12977495690256166,1000.0,0.0,108.33845650786478,0.20912956400118893,-402.470448997188,-0.014507564960606381,53.47444224605571,1.4743744320154013,2.403616200418422,0.0,0.0,50000.0,4.50537683862925,1600000 +0.1288612035514419,1000.0,0.0,108.94506835538063,0.9708749910576171,-407.5964844555719,0.008840826111268043,54.86875132139223,1.4946278975248133,2.418530487629154,0.0,0.0,50000.0,4.502368273937661,1650000 +0.12410909739807241,973.3,131.13218521781752,109.81963239667851,15.948369201593662,-410.4468209417724,0.0006521989998591877,53.08770000825872,1.4456830853557472,2.3775845484459195,0.0,0.0,50000.0,4.503792574564938,1700000 +0.12922628047495985,1000.0,0.0,111.81454408535303,0.16259703727847108,-409.0182661987284,0.002192106536701369,55.80660991418798,1.4864330634379461,2.4304967672300504,0.0,0.0,50000.0,4.5048398952306705,1750000 +0.12837143673270876,1000.0,0.0,114.23698133928903,0.2514123072838161,-409.5709272436506,-0.001961240260712723,55.86949146358486,1.4996326078724835,2.454322374080457,0.0,0.0,50000.0,4.504133059812232,1800000 +0.1268143615373946,1000.0,0.0,113.78442481839267,1.4487661946111499,-411.6643117120402,-0.008111527309827513,54.910697390899806,1.4900914383542565,2.4330059349369537,0.0,0.0,50000.0,4.50237289398531,1850000 +0.12719242991313115,994.29,56.81378265878799,116.17464874016355,7.113695436597489,-410.08096780151635,0.010378438806347328,55.888690892372075,1.484754975898271,2.459806901962709,0.0,0.0,50000.0,4.499932331781436,1900000 +0.12700055356651663,741.53,262.4639577161024,82.00876728141509,32.106525465273435,-409.38511596740074,-0.006925761970467874,53.27084151800126,1.4518394255697575,2.414393520128742,0.0,0.0,50000.0,4.505524819423173,1950000 +0.12605857074409965,1000.0,0.0,114.15553826157789,1.2309401558787176,-410.7221722308365,0.0029782748669759593,54.075763172188296,1.4712127100360601,2.4039747016716437,0.0,0.0,50000.0,4.505271041612542,2000000 +0.1308554568047867,1000.0,0.0,113.28480685557018,1.2174122981585742,-407.88619445556446,0.0028624129312112404,58.05735073839219,1.5368452441334675,2.4828312782954995,0.0,0.0,50000.0,4.508854400958117,2050000 +0.12283841715246538,1000.0,0.0,114.00598703580907,0.4765236309350545,-409.4214220861888,0.005007943780012574,53.83893974386211,1.4788259242141173,2.4100712580681,0.0,0.0,50000.0,4.5061559966505005,2100000 +0.12387971274301507,1000.0,0.0,115.59191637819939,0.2054709289638278,-409.9391599108909,-0.012585945168435576,53.76660593835816,1.4758919297337485,2.4291738528800138,0.0,0.0,50000.0,4.504741080964943,2150000 +0.11823365209832722,1000.0,0.0,114.75448220745012,0.8863884133056672,-411.54346282409443,-0.0030353819552436817,55.39388889690411,1.493958318861717,2.4505913780761324,0.0,0.0,50000.0,4.5057215064190785,2200000 +0.11875331275507921,1000.0,0.0,111.84999693958306,0.28233238436247887,-408.9008234851029,-0.008678969642184845,58.234253180771276,1.5226061163735602,2.5381356572651494,0.0,0.0,50000.0,4.504650495255361,2250000 +0.12146998710855855,1000.0,0.0,111.7144619021186,0.4243945515981849,-407.5176441589333,0.004168370269946773,58.47080987236046,1.5391667416715706,2.5113817358923267,0.0,0.0,50000.0,4.504117748399327,2300000 +0.12243535320058574,1000.0,0.0,112.79642652560824,0.8484102500739734,-406.3687057891816,0.011127605520710257,57.55983416235001,1.5307359663939288,2.491395805315951,0.0,0.0,50000.0,4.504707210143716,2350000 +0.1293900925530494,1000.0,0.0,110.81809402315584,0.27147540741105064,-402.47976843749444,0.0006762111413852275,64.6156163132282,1.6238024068832504,2.6541680229353415,0.0,0.0,50000.0,4.504168850767234,2400000 +0.11991738592237293,1000.0,0.0,115.53848128907185,0.8641736042354446,-403.1343469970649,-0.010073149722553899,62.03355245952639,1.5799064542150532,2.6148349014532295,0.0,0.0,50000.0,4.504184184404877,2450000 +0.12064091637849668,1000.0,0.0,111.64553364397504,0.7520594089903523,-406.32832530822793,0.005571070897467393,61.42872941885013,1.5856716655039813,2.6033072486233757,0.0,0.0,50000.0,4.502236810765057,2500000 +0.12483125427559226,1000.0,0.0,113.36652131466192,0.6610490848252325,-405.33291941589454,-0.0035752887247874595,61.1560249744606,1.5749230751681444,2.612780426514116,0.0,0.0,50000.0,4.503392477758953,2550000 +0.11846706987351308,1000.0,0.0,111.03913714786712,0.3829450732775645,-406.4076131317108,0.00845102695897229,59.80507716116035,1.5440840935313527,2.5855698573994563,0.0,0.0,50000.0,4.498895619658243,2600000 +0.11585969179063992,1000.0,0.0,115.93518150492967,0.24714078359430336,-407.90611198791277,-0.003950798321254392,57.7280024470902,1.5373293166804156,2.5376301039374485,0.0,0.0,50000.0,4.501835302295675,2650000 +0.11654150091364945,1000.0,0.0,115.90902029955026,0.3021485090230826,-402.86855617675417,0.014674708774611212,60.294900366668244,1.5921372346544145,2.5732475822854206,0.0,0.0,50000.0,4.503689831826689,2700000 +0.12769468535035916,1000.0,0.0,114.11354202884061,0.3794642575350339,-400.5649481311047,-0.010246641094721773,62.12152420120226,1.6604396673750894,2.603731920828835,0.0,0.0,50000.0,4.504376876781347,2750000 +0.12611257914423926,1000.0,0.0,118.0220339489463,0.35562353068322894,-401.00689304687694,0.0018494434465093556,65.273413698958,1.6807649913358689,2.6804201805162515,0.0,0.0,50000.0,4.5023520169682545,2800000 +0.12610590447574613,1000.0,0.0,113.00096468089649,0.45727758002135405,-402.80279874267393,-0.014775240989923406,66.3146640061556,1.6769907301950273,2.709458246841408,0.0,0.0,50000.0,4.502442455694581,2850000 +0.11592798831507463,992.51,44.17250162714357,116.4443188821415,5.600498225169978,-404.69440472778166,0.016990728878193008,66.1738492522051,1.659346924576787,2.7143495561003284,0.0,0.0,50000.0,4.502185384412961,2900000 +0.11440565230339794,998.43,15.621302762573931,116.78873916315912,2.0092361790200393,-402.1484210308787,0.011112683654688392,64.76155995720016,1.6316106107878503,2.6987586324620785,0.0,0.0,50000.0,4.507774704345128,2950000 +0.10445701171591827,1000.0,0.0,115.93158482044407,0.36080902119883684,-394.2131348504637,-0.024310212576613244,67.64037447538342,1.6743034931969747,2.738902911310182,0.0,0.0,50000.0,4.503056522892392,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_42×tamp_26-0505-202556/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/kde/seed_42×tamp_26-0505-202556/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..7b94d9a93d4ba9fad9d0b4104c26be3ff9acebf5 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/kde/seed_42×tamp_26-0505-202556/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.5251 +num rollout transitions: 50000, reward mean: 4.5710 +num rollout transitions: 50000, reward mean: 4.5472 +num rollout transitions: 50000, reward mean: 4.5512 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4781 +num rollout transitions: 50000, reward mean: 4.4707 +num rollout transitions: 50000, reward mean: 4.4777 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5314 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.5358 +num rollout transitions: 50000, reward mean: 4.5476 +num rollout transitions: 50000, reward mean: 4.5460 +num rollout transitions: 50000, reward mean: 4.5460 +num rollout transitions: 50000, reward mean: 4.5321 +num rollout transitions: 50000, reward mean: 4.5329 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5398 +num rollout transitions: 50000, reward mean: 4.5332 +num rollout transitions: 50000, reward mean: 4.5217 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5252 +num rollout transitions: 50000, reward mean: 4.5292 +num rollout transitions: 50000, reward mean: 4.5321 +num rollout transitions: 50000, reward mean: 4.5439 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.5301 +num rollout transitions: 50000, reward mean: 4.5346 +num rollout transitions: 50000, reward mean: 4.5367 +num rollout transitions: 50000, reward mean: 4.5277 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5473 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5427 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5257 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5075 +---------------------------------------------------------------------------------- +| alpha | 0.258 | +| eval/episode_length | 22.1 | +| eval/episode_length_std | 18.2 | +| eval/normalized_episode_reward | 0.00319 | +| eval/normalized_episode_reward_std | 0.368 | +| loss/actor | -287 | +| loss/alpha | -1.86 | +| loss/critic | 21.3 | +| penalty/lcb_mean | 1.02 | +| penalty/lcb_std | 1.28 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.52 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4787 +num rollout transitions: 50000, reward mean: 4.4970 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 141 | +| eval/episode_length_std | 34.4 | +| eval/normalized_episode_reward | 1.39 | +| eval/normalized_episode_reward_std | 2.38 | +| loss/actor | -429 | +| loss/alpha | 0.017 | +| loss/critic | 33.3 | +| penalty/lcb_mean | 1.25 | +| penalty/lcb_std | 1.77 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5274 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5264 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4657 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5441 +num rollout transitions: 50000, reward mean: 4.5365 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4774 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5317 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5242 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5321 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 287 | +| eval/episode_length_std | 115 | +| eval/normalized_episode_reward | 18.5 | +| eval/normalized_episode_reward_std | 12.4 | +| loss/actor | -454 | +| loss/alpha | 0.00975 | +| loss/critic | 36.3 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5328 +num rollout transitions: 50000, reward mean: 4.5332 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.5222 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5323 +num rollout transitions: 50000, reward mean: 4.5240 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5263 +num rollout transitions: 50000, reward mean: 4.5266 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5255 +num rollout transitions: 50000, reward mean: 4.5258 +num rollout transitions: 50000, reward mean: 4.5427 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5290 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5300 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5384 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5461 +num rollout transitions: 50000, reward mean: 4.5271 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.5348 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5406 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5178 +---------------------------------------------------------------------------------- +| alpha | 0.137 | +| eval/episode_length | 260 | +| eval/episode_length_std | 53.7 | +| eval/normalized_episode_reward | 18.4 | +| eval/normalized_episode_reward_std | 5.71 | +| loss/actor | -462 | +| loss/alpha | -0.0117 | +| loss/critic | 35.5 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 1.86 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.52 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5276 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5363 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5276 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.5389 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5395 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5312 +num rollout transitions: 50000, reward mean: 4.5319 +num rollout transitions: 50000, reward mean: 4.5349 +num rollout transitions: 50000, reward mean: 4.5282 +num rollout transitions: 50000, reward mean: 4.5426 +num rollout transitions: 50000, reward mean: 4.5384 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5361 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5327 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5283 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5309 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5347 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5291 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5534 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5266 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5129 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 156 | +| eval/episode_length_std | 55.3 | +| eval/normalized_episode_reward | 7.93 | +| eval/normalized_episode_reward_std | 5.16 | +| loss/actor | -430 | +| loss/alpha | 0.00403 | +| loss/critic | 29.9 | +| penalty/lcb_mean | 1.18 | +| penalty/lcb_std | 1.68 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.52 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5398 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5286 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5328 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5268 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5297 +num rollout transitions: 50000, reward mean: 4.5320 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5415 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5317 +num rollout transitions: 50000, reward mean: 4.5284 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5346 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5293 +num rollout transitions: 50000, reward mean: 4.5017 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 103 | +| eval/episode_length_std | 6.71 | +| eval/normalized_episode_reward | 5.2 | +| eval/normalized_episode_reward_std | 0.386 | +| loss/actor | -416 | +| loss/alpha | 0.0146 | +| loss/critic | 32.1 | +| penalty/lcb_mean | 1.23 | +| penalty/lcb_std | 1.71 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.52 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5219 +num rollout transitions: 50000, reward mean: 4.5313 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5287 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5295 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5150 +---------------------------------------------------------------------------------- +| alpha | 0.143 | +| eval/episode_length | 117 | +| eval/episode_length_std | 9.6 | +| eval/normalized_episode_reward | 5.91 | +| eval/normalized_episode_reward_std | 0.917 | +| loss/actor | -411 | +| loss/alpha | -0.0107 | +| loss/critic | 32.7 | +| penalty/lcb_mean | 1.22 | +| penalty/lcb_std | 1.74 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5369 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4760 +num rollout transitions: 50000, reward mean: 4.4667 +num rollout transitions: 50000, reward mean: 4.4826 +---------------------------------------------------------------------------------- +| alpha | 0.143 | +| eval/episode_length | 221 | +| eval/episode_length_std | 262 | +| eval/normalized_episode_reward | 17.1 | +| eval/normalized_episode_reward_std | 26.6 | +| loss/actor | -404 | +| loss/alpha | 0.00112 | +| loss/critic | 34.9 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.8 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5288 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.4794 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5036 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.7 | +| eval/normalized_episode_reward_std | 0.681 | +| loss/actor | -401 | +| loss/alpha | -0.00528 | +| loss/critic | 37 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 1.9 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5272 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4828 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4835 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5263 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5196 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 627 | +| eval/episode_length_std | 249 | +| eval/normalized_episode_reward | 61.5 | +| eval/normalized_episode_reward_std | 28 | +| loss/actor | -399 | +| loss/alpha | 0.00534 | +| loss/critic | 38.4 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 1.92 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4760 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5336 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4804 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4918 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 983 | +| eval/episode_length_std | 85.6 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 10.4 | +| loss/actor | -396 | +| loss/alpha | -0.00128 | +| loss/critic | 39.6 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 1.97 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4823 +num rollout transitions: 50000, reward mean: 4.4792 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4781 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4708 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4843 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5002 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 988 | +| eval/episode_length_std | 68.1 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 8.29 | +| loss/actor | -398 | +| loss/alpha | -0.012 | +| loss/critic | 37.6 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 1.95 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4828 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4817 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4744 +num rollout transitions: 50000, reward mean: 4.4752 +num rollout transitions: 50000, reward mean: 4.4744 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4942 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 115 | +| eval/episode_length_std | 4.07 | +| eval/normalized_episode_reward | 7.23 | +| eval/normalized_episode_reward_std | 0.415 | +| loss/actor | -400 | +| loss/alpha | -0.00601 | +| loss/critic | 38.2 | +| penalty/lcb_mean | 1.28 | +| penalty/lcb_std | 1.97 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5249 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4739 +num rollout transitions: 50000, reward mean: 4.4704 +num rollout transitions: 50000, reward mean: 4.4742 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4816 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4831 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.5297 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5042 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 964 | +| eval/episode_length_std | 109 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 13.1 | +| loss/actor | -395 | +| loss/alpha | 0.00943 | +| loss/critic | 39.6 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 2.01 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5260 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4811 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4716 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5077 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 0.593 | +| loss/actor | -387 | +| loss/alpha | 0.0118 | +| loss/critic | 43.6 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.12 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4817 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4720 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5273 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4743 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4790 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4849 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 0.922 | +| loss/actor | -385 | +| loss/alpha | -0.0221 | +| loss/critic | 44.1 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.1 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4805 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4785 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4822 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5324 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4991 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 974 | +| eval/episode_length_std | 93.5 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 10.3 | +| loss/actor | -387 | +| loss/alpha | 0.0253 | +| loss/critic | 47.3 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.2 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4768 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.4794 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4801 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4720 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4842 +---------------------------------------------------------------------------------- +| alpha | 0.147 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 0.616 | +| loss/actor | -383 | +| loss/alpha | -0.00491 | +| loss/critic | 50.6 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4774 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4756 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.4760 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5179 +---------------------------------------------------------------------------------- +| alpha | 0.147 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 1.19 | +| loss/actor | -380 | +| loss/alpha | 0.00852 | +| loss/critic | 54 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5271 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5273 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4796 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5258 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4805 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4881 +---------------------------------------------------------------------------------- +| alpha | 0.148 | +| eval/episode_length | 986 | +| eval/episode_length_std | 73 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 8.48 | +| loss/actor | -382 | +| loss/alpha | -0.0122 | +| loss/critic | 54.5 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.37 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4831 +num rollout transitions: 50000, reward mean: 4.4782 +num rollout transitions: 50000, reward mean: 4.4782 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.4790 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4773 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4816 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.4840 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5029 +---------------------------------------------------------------------------------- +| alpha | 0.136 | +| eval/episode_length | 994 | +| eval/episode_length_std | 46.3 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 5.96 | +| loss/actor | -386 | +| loss/alpha | 0.00306 | +| loss/critic | 50.3 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.27 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4837 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5331 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5064 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 923 | +| eval/episode_length_std | 167 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 20 | +| loss/actor | -387 | +| loss/alpha | 0.00247 | +| loss/critic | 54.6 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5226 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4785 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5266 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4819 +num rollout transitions: 50000, reward mean: 4.4803 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4814 +num rollout transitions: 50000, reward mean: 4.4792 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4790 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5046 +---------------------------------------------------------------------------------- +| alpha | 0.143 | +| eval/episode_length | 981 | +| eval/episode_length_std | 91 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 11 | +| loss/actor | -385 | +| loss/alpha | 0.00223 | +| loss/critic | 56.3 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4847 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4770 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4797 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4775 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4774 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4768 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5003 +----------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.792 | +| loss/actor | -383 | +| loss/alpha | -0.000426 | +| loss/critic | 55.2 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1200000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4817 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4832 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4749 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4807 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5128 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 1.47 | +| loss/actor | -379 | +| loss/alpha | 0.00551 | +| loss/critic | 56.7 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4819 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4824 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4778 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5038 +---------------------------------------------------------------------------------- +| alpha | 0.144 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.939 | +| loss/actor | -381 | +| loss/alpha | -0.0023 | +| loss/critic | 57.1 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.44 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5241 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4831 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5222 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4811 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4954 +---------------------------------------------------------------------------------- +| alpha | 0.142 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.224 | +| loss/actor | -378 | +| loss/alpha | -0.00749 | +| loss/critic | 58.7 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.47 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4812 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4802 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4698 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.4800 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.4912 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 0.683 | +| loss/actor | -379 | +| loss/alpha | 0.00286 | +| loss/critic | 58.4 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.5 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4795 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.4817 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4833 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.4743 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5049 +---------------------------------------------------------------------------------- +| alpha | 0.142 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.158 | +| loss/actor | -374 | +| loss/alpha | 0.00309 | +| loss/critic | 59.8 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.51 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4745 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4787 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4835 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5217 +num rollout transitions: 50000, reward mean: 4.4912 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 2.52 | +| loss/actor | -375 | +| loss/alpha | -0.00732 | +| loss/critic | 55.7 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.43 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5288 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5251 +num rollout transitions: 50000, reward mean: 4.5272 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4781 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5022 +----------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 1.25 | +| loss/actor | -375 | +| loss/alpha | -0.000451 | +| loss/critic | 56.3 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1550000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5242 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5245 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4993 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 2.34 | +| loss/actor | -376 | +| loss/alpha | 0.00743 | +| loss/critic | 56.8 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5295 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5234 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4997 +---------------------------------------------------------------------------------- +| alpha | 0.142 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.414 | +| loss/actor | -375 | +| loss/alpha | -0.00337 | +| loss/critic | 57.7 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5263 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4789 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4994 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.52 | +| loss/actor | -375 | +| loss/alpha | 0.00397 | +| loss/critic | 58.5 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5289 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4807 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4972 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 0.424 | +| loss/actor | -380 | +| loss/alpha | -0.00974 | +| loss/critic | 58.3 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5067 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.508 | +| loss/actor | -383 | +| loss/alpha | 0.0029 | +| loss/critic | 54.7 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5313 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5281 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5059 +---------------------------------------------------------------------------------- +| alpha | 0.142 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 1.9 | +| loss/actor | -379 | +| loss/alpha | 0.0106 | +| loss/critic | 58.7 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.4799 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5264 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5100 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.95 | +| loss/actor | -380 | +| loss/alpha | -0.00149 | +| loss/critic | 57.5 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.43 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.5205 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5290 +num rollout transitions: 50000, reward mean: 4.5219 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5323 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5053 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.966 | +| loss/actor | -383 | +| loss/alpha | 0.00168 | +| loss/critic | 60.9 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4820 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5050 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 2.88 | +| loss/actor | -382 | +| loss/alpha | -0.00169 | +| loss/critic | 62.3 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.57 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5044 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.839 | +| loss/actor | -377 | +| loss/alpha | 0.00178 | +| loss/critic | 59.4 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5260 +num rollout transitions: 50000, reward mean: 4.5223 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5242 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4965 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 1.11 | +| loss/actor | -382 | +| loss/alpha | -0.012 | +| loss/critic | 56.7 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5226 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5241 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5255 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4816 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5243 +---------------------------------------------------------------------------------- +| alpha | 0.136 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.753 | +| loss/actor | -383 | +| loss/alpha | 0.0136 | +| loss/critic | 60.6 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.52 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5346 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4815 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5263 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5112 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 0.301 | +| loss/actor | -381 | +| loss/alpha | -0.0196 | +| loss/critic | 60.4 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.53 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5299 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4729 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4974 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.207 | +| loss/actor | -384 | +| loss/alpha | 0.00928 | +| loss/critic | 59.2 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4776 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4824 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5081 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.765 | +| loss/actor | -388 | +| loss/alpha | -0.0033 | +| loss/critic | 58.4 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.53 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4825 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4795 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.4927 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.411 | +| loss/actor | -384 | +| loss/alpha | 0.00771 | +| loss/critic | 59.7 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.52 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5075 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.448 | +| loss/actor | -381 | +| loss/alpha | 0.00516 | +| loss/critic | 63.5 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5305 +num rollout transitions: 50000, reward mean: 4.5287 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4977 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.471 | +| loss/actor | -384 | +| loss/alpha | -0.0075 | +| loss/critic | 61.5 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.57 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5271 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5267 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5304 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5226 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.4970 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.45 | +| loss/actor | -383 | +| loss/alpha | 0.00259 | +| loss/critic | 63.1 | +| penalty/lcb_mean | 1.69 | +| penalty/lcb_std | 2.62 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4957 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.442 | +| loss/actor | -386 | +| loss/alpha | -0.017 | +| loss/critic | 62.8 | +| penalty/lcb_mean | 1.71 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4811 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.4820 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5245 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4985 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.401 | +| loss/actor | -387 | +| loss/alpha | 0.0155 | +| loss/critic | 63.4 | +| penalty/lcb_mean | 1.71 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4822 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4800 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4814 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5158 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 0.734 | +| loss/actor | -386 | +| loss/alpha | 0.0075 | +| loss/critic | 62.5 | +| penalty/lcb_mean | 1.69 | +| penalty/lcb_std | 2.62 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5027 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.585 | +| loss/actor | -386 | +| loss/alpha | -0.0293 | +| loss/critic | 63 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.64 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5279 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 955 | +| eval/episode_length_std | 110 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 13 | +| loss/actor | -385 | +| loss/alpha | 0.0145 | +| loss/critic | 61.9 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5013 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.996 | +| loss/actor | -381 | +| loss/alpha | 0.00248 | +| loss/critic | 63.8 | +| penalty/lcb_mean | 1.71 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5034 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 998 | +| eval/episode_length_std | 22.1 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 2.87 | +| loss/actor | -380 | +| loss/alpha | -0.0161 | +| loss/critic | 64.3 | +| penalty/lcb_mean | 1.73 | +| penalty/lcb_std | 2.66 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5004 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 995 | +| eval/episode_length_std | 49.7 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 6.32 | +| loss/actor | -380 | +| loss/alpha | 0.00419 | +| loss/critic | 62.5 | +| penalty/lcb_mean | 1.73 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5014 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.543 | +| loss/actor | -381 | +| loss/alpha | 0.0144 | +| loss/critic | 64.7 | +| penalty/lcb_mean | 1.75 | +| penalty/lcb_std | 2.67 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5010 +---------------------------------------------------------------------------------- +| alpha | 0.113 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.627 | +| loss/actor | -377 | +| loss/alpha | -0.0325 | +| loss/critic | 67 | +| penalty/lcb_mean | 1.8 | +| penalty/lcb_std | 2.72 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 58351.20s diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_42×tamp_26-0505-202556/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/kde/seed_42×tamp_26-0505-202556/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_42×tamp_26-0505-202556/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/kde/seed_42×tamp_26-0505-202556/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..41893ca1167d952a82921a998b4fd06dede558c4 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/kde/seed_42×tamp_26-0505-202556/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 42, "device": "cuda:4", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_42×tamp_26-0112-030600/model", "reward_penalty_coef": 0.3, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/kde", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "kde", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "kde"} diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_42×tamp_26-0505-202556/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/kde/seed_42×tamp_26-0505-202556/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..9bbf9dc60d26b817d7be75ea54732a072951f767 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/kde/seed_42×tamp_26-0505-202556/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.25756618881955706,22.06,18.22021953764553,0.0031880513153489266,0.3678463631006602,-286.50817630776,-1.8565046234163403,21.260095095193083,1.0223303939878519,1.278098288613513,0.0,0.0,50000.0,4.5216513162769525,50000 +0.1306329577510019,141.11,34.35982974346642,1.3916243976767015,2.381151545802946,-428.8821249505603,0.01695167460061622,33.274136470432786,1.2547051858616145,1.7725668053436339,0.0,0.0,50000.0,4.507121076311059,100000 +0.13549800427794345,287.11,114.71529061114738,18.464361994500155,12.380615290930324,-454.29999601501606,0.009747948399297893,36.254683168372324,1.314458744561667,1.879484859480862,0.0,0.0,50000.0,4.510668070720825,150000 +0.1369676144707199,260.31,53.7018984766833,18.38479055614127,5.713863631075992,-461.7238716107168,-0.011676220362782262,35.45925379095085,1.3270413216626291,1.8551735852897269,0.0,0.0,50000.0,4.5179983498097185,200000 +0.13483977065175823,156.07,55.270291296500325,7.928593082352848,5.162049624198484,-429.5577924822961,0.004027847991473992,29.86254436077114,1.1823296212160712,1.6762570739614953,0.0,0.0,50000.0,4.520487072393205,250000 +0.1386607318992938,102.68,6.708025044675965,5.199453542596647,0.3856323271440219,-416.02059834045343,0.014582065129391917,32.13539910779898,1.2287287834334277,1.7060603338754374,0.0,0.0,50000.0,4.515098331231612,300000 +0.1434072859349872,117.29,9.604472916303111,5.9118465886769656,0.9170997235999326,-411.3162230822747,-0.010703628516383368,32.65218090389225,1.2196778700709396,1.7364589993453028,0.0,0.0,50000.0,4.510500196185802,350000 +0.14265907851368306,220.94,261.9407116123799,17.10087615003531,26.63428256935237,-404.4078269220014,0.0011171305648981827,34.943468604888906,1.2681144368195196,1.7991556084680498,0.0,0.0,50000.0,4.5049708754567,400000 +0.13968659315764784,1000.0,0.0,99.67211370371672,0.6808036607462686,-401.24819957702215,-0.00527890736013665,37.001069420089756,1.292917516124246,1.8975861090564954,0.0,0.0,50000.0,4.501695791454699,450000 +0.13991117211103257,627.4,249.13662918166008,61.51145384098186,28.032846202331207,-398.6573783630369,0.005344061549194263,38.430740366154254,1.3040016244077774,1.9151939867615668,0.0,0.0,50000.0,4.504250833026898,500000 +0.1390731691566124,983.14,85.62161175777993,101.04649755615054,10.410648798078942,-396.24131392883214,-0.0012823971495030879,39.60379170024938,1.317198116481283,1.9701931811904896,0.0,0.0,50000.0,4.502205993243833,550000 +0.1312721324646478,988.22,68.14786570392356,104.8769908690911,8.286955718769908,-398.1849763561997,-0.012025183758698486,37.645497954558735,1.2617993702006616,1.952244657317384,0.0,0.0,50000.0,4.499299891046531,600000 +0.12832030675768713,114.63,4.066091489378958,7.231139007726307,0.41528430811237926,-399.7113899328553,-0.006014766785092687,38.18240558843565,1.2795095763409252,1.9682979666078242,0.0,0.0,50000.0,4.495999914109416,650000 +0.12931782240748477,963.68,109.35948792857435,105.04412042145745,13.089322666117958,-395.2329870318699,0.009427283490430682,39.557166464195554,1.2956596363365513,2.0068225632178622,0.0,0.0,50000.0,4.497833912327713,700000 +0.13764205655634038,1000.0,0.0,104.72620770658285,0.5927276685800082,-387.4306174096689,0.011791838882379304,43.59661092376712,1.3747689602816175,2.116415919536347,0.0,0.0,50000.0,4.500440583923056,750000 +0.1351022831402723,1000.0,0.0,107.22375656786618,0.9222408657433077,-384.5059275592051,-0.022054100931286787,44.061488523883405,1.3803339890027047,2.099752920911282,0.0,0.0,50000.0,4.498680248500111,800000 +0.1383580456744148,973.9,93.47892810681986,105.71649284925475,10.259530802430278,-387.48959559631345,0.0252814120550826,47.315747698097375,1.421143535666478,2.1953017860555453,0.0,0.0,50000.0,4.502340171488149,850000 +0.1470624483260529,1000.0,0.0,107.0409867391572,0.615875894839563,-382.59018695190275,-0.004911900546550834,50.55642108177121,1.4966926522922528,2.2403453610384703,0.0,0.0,50000.0,4.4964958129370896,900000 +0.1471056401723629,1000.0,0.0,110.75034310999786,1.1863930671142693,-380.0653139874323,0.008520132478922688,53.95537204137761,1.524281447074413,2.353766587915408,0.0,0.0,50000.0,4.498480800512961,950000 +0.14764692501425894,985.68,73.00984591135638,106.17971698508117,8.484639404797315,-382.02364858093273,-0.01216569626942273,54.484076700515516,1.5343099600970806,2.3697303094947517,0.0,0.0,50000.0,4.50104074908763,1000000 +0.13603949576795207,994.45,46.25416197489691,113.92953121617404,5.962068416948059,-385.53387456420813,0.0030590732597931786,50.30783089694916,1.4933241016936196,2.271612702811927,0.0,0.0,50000.0,4.495950097161425,1050000 +0.14129133165955296,923.13,166.71824465246752,101.82996355082672,19.95631558860024,-386.9668629760735,0.002469671401195187,54.564358003235,1.5471311771726717,2.3775843836903117,0.0,0.0,50000.0,4.499616174390673,1100000 +0.14322759398013668,981.44,91.01157289048464,108.19714810299979,11.040262291131022,-384.8368549517782,0.002231980619393277,56.34117127229725,1.5819705419826366,2.422900792534363,0.0,0.0,50000.0,4.497877151292946,1150000 +0.13857822988837948,1000.0,0.0,109.96295459212153,0.7920192412033129,-382.76003278748016,-0.00042643583565950705,55.17839830896285,1.5477539301347731,2.4079469801092745,0.0,0.0,50000.0,4.497625476748506,1200000 +0.14118721494764058,1000.0,0.0,105.17133334872295,1.4704195872753685,-379.36790142577905,0.0055051848825439995,56.6812066342739,1.592956705584518,2.408602819533369,0.0,0.0,50000.0,4.498915025906879,1250000 +0.14446956563711208,1000.0,0.0,112.46503034805716,0.9389720329875229,-380.8714073669441,-0.0022972759703919765,57.079580852394606,1.6096998368358704,2.436553881194631,0.0,0.0,50000.0,4.497410518652785,1300000 +0.14171429166584934,1000.0,0.0,112.61558572018977,0.22396450527453754,-377.9510080926528,-0.007485434556789736,58.665056731911015,1.6070810669279094,2.4671649126601216,0.0,0.0,50000.0,4.496502000036304,1350000 +0.13506764709979296,1000.0,0.0,105.38561525022186,0.683027759792062,-379.1077750134225,0.0028555199223384642,58.41249828329152,1.5875535675334986,2.5031385259508814,0.0,0.0,50000.0,4.497296814240563,1400000 +0.1419560785743589,1000.0,0.0,113.30162319353983,0.15812531809768623,-374.1285715722654,0.003094022858291823,59.83915798080451,1.62606696289302,2.5139427348423284,0.0,0.0,50000.0,4.497814016609996,1450000 +0.1398549040478499,1000.0,0.0,112.44769149727883,2.5237510285895195,-375.2072422265611,-0.007318625664189268,55.73800380336697,1.566956051845539,2.4292940685630326,0.0,0.0,50000.0,4.500343686133531,1500000 +0.13327385110408121,1000.0,0.0,107.61715026918017,1.2452699162002154,-374.7021772790542,-0.0004513633003458376,56.31357914712857,1.5783642634201311,2.4222771022224787,0.0,0.0,50000.0,4.506810834329998,1550000 +0.14131193588077842,1000.0,0.0,105.91358333951631,2.336069617273822,-375.9593537518317,0.007426471725366845,56.75822374609016,1.6078962486982498,2.417866869356643,0.0,0.0,50000.0,4.504978119527172,1600000 +0.1421841816502823,1000.0,0.0,109.857634564832,0.41443437988771753,-374.6157888531487,-0.0033693792468308488,57.727687487640644,1.6368159910893354,2.4493577652692546,0.0,0.0,50000.0,4.504136894831894,1650000 +0.14536045097380604,1000.0,0.0,112.87910532520466,0.5203752663538994,-375.4940026824971,0.003968999060802205,58.519193240699465,1.6412029105591743,2.45453422417881,0.0,0.0,50000.0,4.503888716182397,1700000 +0.14134653395861363,1000.0,0.0,106.41063664238786,0.4240491859225269,-379.5204234484873,-0.00973597424108543,58.25153017030721,1.6188944673585814,2.462098126268397,0.0,0.0,50000.0,4.503732365183331,1750000 +0.13314589959889786,1000.0,0.0,109.77140338058706,0.5075773434577591,-383.44494003845625,0.002895848561264494,54.73490163303343,1.5706241519307809,2.415619823864678,0.0,0.0,50000.0,4.506395775865865,1800000 +0.14172407843977244,1000.0,0.0,108.15726117539717,1.8964185400623608,-378.95146651549834,0.01059777837729087,58.710386750888716,1.6522767861890926,2.4620162572169675,0.0,0.0,50000.0,4.508179282982554,1850000 +0.14137374386817161,1000.0,0.0,113.69285295370506,1.9511093806592847,-380.0778606372124,-0.0014862142815253402,57.50818623237642,1.6236844713878644,2.433523922924982,0.0,0.0,50000.0,4.504678733605761,1900000 +0.1448069124275482,1000.0,0.0,114.13876459815822,0.9660674204428803,-383.4080370465179,0.001683233570940853,60.9182137014387,1.6801842550563717,2.542053404800863,0.0,0.0,50000.0,4.5081241218442445,1950000 +0.13921661955863252,1000.0,0.0,109.81951880506271,2.8788949627395906,-382.03883791748,-0.0016945933459699227,62.33276682746896,1.6666564981984873,2.570248570203761,0.0,0.0,50000.0,4.50369291177854,2000000 +0.13953597433060472,1000.0,0.0,113.19802314162024,0.8390390397330293,-377.111481138304,0.0017782246959208475,59.44781822988481,1.650238430325966,2.494524843511596,0.0,0.0,50000.0,4.5085580713263935,2050000 +0.13518766525209222,1000.0,0.0,115.40012556191428,1.1063301882832717,-381.8602719036874,-0.012040258307754804,56.694834709606695,1.6166514603996254,2.4215143802690666,0.0,0.0,50000.0,4.508120114340368,2100000 +0.13606437278866518,1000.0,0.0,114.60838676460017,0.7527857483076471,-383.49421526733835,0.013562192440740705,60.64097625185077,1.664333143951872,2.522127774630834,0.0,0.0,50000.0,4.507822511968474,2150000 +0.13200794077456132,1000.0,0.0,105.16202344886409,0.3009683375681002,-381.34906508911183,-0.019595253829695312,60.36726602590524,1.671465106596947,2.5290042165184166,0.0,0.0,50000.0,4.505741793540165,2200000 +0.13122358766048903,1000.0,0.0,115.78496863079917,0.2074488330368916,-383.5835949688749,0.009284351842776027,59.17606932786973,1.6482370634174481,2.537212075297838,0.0,0.0,50000.0,4.502315738015976,2250000 +0.12647346287071665,1000.0,0.0,115.29495581753125,0.7654757646935477,-387.94925832153126,-0.003300793491527384,58.43867476179125,1.6182619304561654,2.5347709226131445,0.0,0.0,50000.0,4.503260479749511,2300000 +0.1314617817123214,1000.0,0.0,115.77297863643051,0.41057474480815587,-383.9578102990688,0.007708935503028393,59.71348032894153,1.6594721686029457,2.519403708929989,0.0,0.0,50000.0,4.503875909902681,2350000 +0.13287448077693617,1000.0,0.0,114.39363629697388,0.44849095528221317,-381.4147202264408,0.005155285145006971,63.47506818332704,1.7043068375611288,2.585832825906294,0.0,0.0,50000.0,4.506027317801247,2400000 +0.1309332633849996,1000.0,0.0,112.48942784686746,0.4710481229057209,-383.52567576782246,-0.007503158894553785,61.453688431320536,1.6844651296234077,2.565674441380538,0.0,0.0,50000.0,4.508559694720464,2450000 +0.13215602951526553,1000.0,0.0,113.74380473341301,1.451192816175224,-383.0745499676529,0.002588122441172702,63.1153700663368,1.6860332735824703,2.6218326164531347,0.0,0.0,50000.0,4.508682834144587,2500000 +0.13365792488828623,1000.0,0.0,115.92643774089551,0.44229159697849174,-385.85994835571245,-0.017043136403020413,62.7819178957177,1.7091265782761556,2.6123352760672804,0.0,0.0,50000.0,4.5050878575614615,2550000 +0.12789119493916476,1000.0,0.0,113.64647136991732,0.4006192894595555,-386.68276914856546,0.015450945632680721,63.39305623872736,1.7141907175874531,2.6123772844934763,0.0,0.0,50000.0,4.502172237316201,2600000 +0.12618890472113992,1000.0,0.0,118.10088848264657,0.7335495533279888,-385.6816851550252,0.007502180959545143,62.49403660041857,1.6919371764302336,2.615076785180593,0.0,0.0,50000.0,4.502824413206164,2650000 +0.12575481238111896,1000.0,0.0,116.34878997849354,0.5849445238404135,-385.84411462768037,-0.029272270808070707,62.953646233253714,1.7004643585777064,2.64300681316853,0.0,0.0,50000.0,4.504333539926818,2700000 +0.1162051959639804,954.93,109.71519994968793,107.75679931395568,12.968946295848957,-385.09590975891456,0.014501932848394095,61.86170432010699,1.683463909039504,2.6267755601215295,0.0,0.0,50000.0,4.502592333175681,2750000 +0.12575199263587705,1000.0,0.0,113.91022053896525,0.9955258952054516,-381.3834141973905,0.002480242630355237,63.773928304786345,1.7100953533411096,2.651372893128417,0.0,0.0,50000.0,4.5028591422737785,2800000 +0.12689056753963318,997.78,22.088721103766957,114.77734890442217,2.865406650575007,-379.90234428405574,-0.016076126438118368,64.2807266875081,1.7347243649435065,2.6639896143007387,0.0,0.0,50000.0,4.504188547743651,2850000 +0.1155261425387855,995.0,49.749371855331,115.31277664638154,6.323857930418075,-379.7858458764631,0.004189237971194011,62.49712669597646,1.726020216617572,2.613366571197488,0.0,0.0,50000.0,4.502443273717696,2900000 +0.12143501054734074,1000.0,0.0,117.4834144550491,0.5425900675117147,-380.95883749572977,0.014430103590972708,64.71636702930441,1.749152254102234,2.6694100306582356,0.0,0.0,50000.0,4.504638371726223,2950000 +0.11328971619620963,1000.0,0.0,116.8406205030494,0.6272519847675619,-377.195098562015,-0.032501199571713574,66.99329317932148,1.8047987625098374,2.71641060158255,0.0,0.0,50000.0,4.504561788381911,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_456×tamp_26-0505-202557/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/kde/seed_456×tamp_26-0505-202557/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a42b804d16e3559d54fc674fe7618e7691cfed --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/kde/seed_456×tamp_26-0505-202557/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 4.4105 +num rollout transitions: 50000, reward mean: 4.5276 +num rollout transitions: 50000, reward mean: 4.5686 +num rollout transitions: 50000, reward mean: 4.5568 +num rollout transitions: 50000, reward mean: 4.5536 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5378 +num rollout transitions: 50000, reward mean: 4.5223 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5358 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5548 +num rollout transitions: 50000, reward mean: 4.5492 +num rollout transitions: 50000, reward mean: 4.5296 +num rollout transitions: 50000, reward mean: 4.5252 +num rollout transitions: 50000, reward mean: 4.5408 +num rollout transitions: 50000, reward mean: 4.5308 +num rollout transitions: 50000, reward mean: 4.5463 +num rollout transitions: 50000, reward mean: 4.5289 +num rollout transitions: 50000, reward mean: 4.5302 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5270 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5385 +num rollout transitions: 50000, reward mean: 4.5304 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5371 +num rollout transitions: 50000, reward mean: 4.5316 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5400 +num rollout transitions: 50000, reward mean: 4.5363 +num rollout transitions: 50000, reward mean: 4.5382 +num rollout transitions: 50000, reward mean: 4.5432 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5386 +num rollout transitions: 50000, reward mean: 4.5450 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5290 +---------------------------------------------------------------------------------- +| alpha | 0.263 | +| eval/episode_length | 114 | +| eval/episode_length_std | 8.34 | +| eval/normalized_episode_reward | -0.37 | +| eval/normalized_episode_reward_std | 0.116 | +| loss/actor | -297 | +| loss/alpha | -1.65 | +| loss/critic | 21 | +| penalty/lcb_mean | 1 | +| penalty/lcb_std | 1.27 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.53 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5240 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5274 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5358 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5363 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5240 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5319 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4738 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5151 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 143 | +| eval/episode_length_std | 117 | +| eval/normalized_episode_reward | 8.86 | +| eval/normalized_episode_reward_std | 10.3 | +| loss/actor | -434 | +| loss/alpha | -0.00348 | +| loss/critic | 26.5 | +| penalty/lcb_mean | 1.04 | +| penalty/lcb_std | 1.55 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5226 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5350 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5187 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 127 | +| eval/episode_length_std | 27.7 | +| eval/normalized_episode_reward | 8.06 | +| eval/normalized_episode_reward_std | 3.25 | +| loss/actor | -435 | +| loss/alpha | 0.0129 | +| loss/critic | 25.2 | +| penalty/lcb_mean | 1.06 | +| penalty/lcb_std | 1.59 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4775 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4784 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5364 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5244 +num rollout transitions: 50000, reward mean: 4.4833 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5247 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 127 | +| eval/episode_length_std | 51.9 | +| eval/normalized_episode_reward | 5.29 | +| eval/normalized_episode_reward_std | 3.58 | +| loss/actor | -430 | +| loss/alpha | 0.024 | +| loss/critic | 29 | +| penalty/lcb_mean | 1.17 | +| penalty/lcb_std | 1.62 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5217 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5301 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5249 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.4801 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5219 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5255 +---------------------------------------------------------------------------------- +| alpha | 0.146 | +| eval/episode_length | 491 | +| eval/episode_length_std | 334 | +| eval/normalized_episode_reward | 36.8 | +| eval/normalized_episode_reward_std | 33.2 | +| loss/actor | -418 | +| loss/alpha | -0.00593 | +| loss/critic | 30.7 | +| penalty/lcb_mean | 1.17 | +| penalty/lcb_std | 1.66 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5256 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5356 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5205 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5279 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5279 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5240 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4817 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5267 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.4768 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5088 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 797 | +| eval/episode_length_std | 361 | +| eval/normalized_episode_reward | 74 | +| eval/normalized_episode_reward_std | 37 | +| loss/actor | -414 | +| loss/alpha | -0.0171 | +| loss/critic | 31.7 | +| penalty/lcb_mean | 1.17 | +| penalty/lcb_std | 1.74 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4790 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5263 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5268 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5267 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5095 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 98.6 | +| eval/normalized_episode_reward_std | 1.77 | +| loss/actor | -410 | +| loss/alpha | 0.00827 | +| loss/critic | 32.8 | +| penalty/lcb_mean | 1.18 | +| penalty/lcb_std | 1.77 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5377 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5260 +num rollout transitions: 50000, reward mean: 4.5219 +num rollout transitions: 50000, reward mean: 4.5348 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.5288 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5331 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5165 +---------------------------------------------------------------------------------- +| alpha | 0.144 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 4.78 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 1.88 | +| loss/actor | -406 | +| loss/alpha | 0.000269 | +| loss/critic | 35.3 | +| penalty/lcb_mean | 1.22 | +| penalty/lcb_std | 1.83 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5249 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5260 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4977 +---------------------------------------------------------------------------------- +| alpha | 0.143 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 97.3 | +| eval/normalized_episode_reward_std | 2.12 | +| loss/actor | -404 | +| loss/alpha | 0.0052 | +| loss/critic | 37.1 | +| penalty/lcb_mean | 1.25 | +| penalty/lcb_std | 1.9 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4847 +num rollout transitions: 50000, reward mean: 4.4814 +num rollout transitions: 50000, reward mean: 4.5266 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4824 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5017 +---------------------------------------------------------------------------------- +| alpha | 0.146 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 2.16 | +| loss/actor | -399 | +| loss/alpha | -0.00504 | +| loss/critic | 40.1 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 2.02 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5302 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5251 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5286 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5035 +---------------------------------------------------------------------------------- +| alpha | 0.153 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 100 | +| eval/normalized_episode_reward_std | 1.41 | +| loss/actor | -396 | +| loss/alpha | 0.00636 | +| loss/critic | 40.3 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.01 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4793 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5325 +num rollout transitions: 50000, reward mean: 4.5147 +---------------------------------------------------------------------------------- +| alpha | 0.155 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 1.64 | +| loss/actor | -393 | +| loss/alpha | 0.00934 | +| loss/critic | 43.8 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.11 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5256 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4796 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4795 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4944 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 0.494 | +| loss/actor | -394 | +| loss/alpha | -0.0169 | +| loss/critic | 41.3 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.06 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4857 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4669 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4783 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4778 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4787 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5287 +num rollout transitions: 50000, reward mean: 4.4788 +num rollout transitions: 50000, reward mean: 4.4910 +---------------------------------------------------------------------------------- +| alpha | 0.142 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 0.546 | +| loss/actor | -392 | +| loss/alpha | -0.00525 | +| loss/critic | 42.4 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.09 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4843 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5249 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4791 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4732 +num rollout transitions: 50000, reward mean: 4.4814 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5081 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 2.47 | +| loss/actor | -391 | +| loss/alpha | 0.013 | +| loss/critic | 40 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 2.02 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4832 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4828 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4753 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5206 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.27 | +| loss/actor | -395 | +| loss/alpha | -0.0102 | +| loss/critic | 40.9 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.07 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4819 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5310 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5180 +---------------------------------------------------------------------------------- +| alpha | 0.136 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.846 | +| loss/actor | -396 | +| loss/alpha | 0.00346 | +| loss/critic | 40.4 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.07 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4789 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.5009 +---------------------------------------------------------------------------------- +| alpha | 0.137 | +| eval/episode_length | 997 | +| eval/episode_length_std | 30.4 | +| eval/normalized_episode_reward | 41.6 | +| eval/normalized_episode_reward_std | 31.4 | +| loss/actor | -394 | +| loss/alpha | -0.00426 | +| loss/critic | 42.8 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.13 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4804 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4778 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4835 +num rollout transitions: 50000, reward mean: 4.4817 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4751 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4840 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4823 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5063 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 2.38 | +| loss/actor | -396 | +| loss/alpha | 0.00086 | +| loss/critic | 43 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.16 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4791 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4765 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5020 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 1.37 | +| loss/actor | -395 | +| loss/alpha | 0.00265 | +| loss/critic | 42.1 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.11 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4805 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.4800 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4791 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5261 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5096 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 1.68 | +| loss/actor | -394 | +| loss/alpha | -0.00724 | +| loss/critic | 41.2 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 2.1 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4787 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4816 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4811 +num rollout transitions: 50000, reward mean: 4.4753 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4835 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.4780 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.384 | +| loss/actor | -395 | +| loss/alpha | 0.00385 | +| loss/critic | 41.5 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 2.11 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4781 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.4767 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4781 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4847 +num rollout transitions: 50000, reward mean: 4.5098 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 1.12 | +| loss/actor | -399 | +| loss/alpha | 0.00501 | +| loss/critic | 42.1 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.12 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4796 +num rollout transitions: 50000, reward mean: 4.5076 +----------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 915 | +| eval/episode_length_std | 162 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 19 | +| loss/actor | -400 | +| loss/alpha | -0.000818 | +| loss/critic | 43.9 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.17 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1200000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4817 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4823 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5278 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.5052 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 0.827 | +| loss/actor | -398 | +| loss/alpha | -0.00311 | +| loss/critic | 45 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.16 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4826 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4847 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4791 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5240 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5205 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5169 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.591 | +| loss/actor | -394 | +| loss/alpha | 0.00726 | +| loss/critic | 43.9 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.16 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4820 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4816 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5275 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4822 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5155 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 2.38 | +| loss/actor | -395 | +| loss/alpha | -0.00501 | +| loss/critic | 46.2 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4783 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4719 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5307 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4772 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4784 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5066 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.274 | +| loss/actor | -395 | +| loss/alpha | -0.00492 | +| loss/critic | 48.4 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.5244 +num rollout transitions: 50000, reward mean: 4.4819 +num rollout transitions: 50000, reward mean: 4.4814 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4815 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4815 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5063 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.517 | +| loss/actor | -395 | +| loss/alpha | 0.00294 | +| loss/critic | 46.3 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.25 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4771 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4782 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4800 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4746 +num rollout transitions: 50000, reward mean: 4.4840 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5104 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.487 | +| loss/actor | -394 | +| loss/alpha | -0.00331 | +| loss/critic | 47.5 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.26 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4782 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4788 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4779 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5253 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5061 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.385 | +| loss/actor | -393 | +| loss/alpha | 0.00366 | +| loss/critic | 50.3 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4804 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4797 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4771 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5261 +num rollout transitions: 50000, reward mean: 4.4766 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4883 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.323 | +| loss/actor | -394 | +| loss/alpha | -0.00622 | +| loss/critic | 50.5 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5242 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5305 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4737 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5207 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.452 | +| loss/actor | -394 | +| loss/alpha | 0.0065 | +| loss/critic | 54.3 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.44 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5363 +num rollout transitions: 50000, reward mean: 4.5256 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4824 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5244 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5244 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4828 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5089 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.57 | +| loss/actor | -392 | +| loss/alpha | 0.00182 | +| loss/critic | 52.2 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.4798 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4769 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.4915 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.852 | +| loss/actor | -393 | +| loss/alpha | -0.00785 | +| loss/critic | 52.6 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5273 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4832 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4779 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4739 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4785 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5110 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.652 | +| loss/actor | -396 | +| loss/alpha | 0.00524 | +| loss/critic | 52.1 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5270 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5292 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4772 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5141 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.47 | +| loss/actor | -399 | +| loss/alpha | -0.0046 | +| loss/critic | 51.3 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4835 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4815 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.4798 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.827 | +| loss/actor | -399 | +| loss/alpha | -0.00197 | +| loss/critic | 50.5 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.36 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4826 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4694 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4794 +num rollout transitions: 50000, reward mean: 4.4656 +num rollout transitions: 50000, reward mean: 4.4766 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5282 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5077 +----------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 996 | +| eval/episode_length_std | 26.3 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 3.44 | +| loss/actor | -399 | +| loss/alpha | -0.000629 | +| loss/critic | 48.7 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1950000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5257 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4835 +num rollout transitions: 50000, reward mean: 4.4769 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4746 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.4817 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5007 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 957 | +| eval/episode_length_std | 148 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 18.4 | +| loss/actor | -396 | +| loss/alpha | 0.0174 | +| loss/critic | 52.3 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4729 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4824 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4738 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4841 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.65 | +| loss/actor | -396 | +| loss/alpha | -0.0126 | +| loss/critic | 55.9 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4847 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5251 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5282 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4796 +num rollout transitions: 50000, reward mean: 4.4843 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5062 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.43 | +| loss/actor | -398 | +| loss/alpha | 0.00211 | +| loss/critic | 53.1 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.44 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.4774 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5241 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4919 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.507 | +| loss/actor | -397 | +| loss/alpha | 6.3e-05 | +| loss/critic | 54.5 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.4770 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4784 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4793 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4997 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.539 | +| loss/actor | -399 | +| loss/alpha | 0.00249 | +| loss/critic | 53.3 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.4827 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5244 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 119 | +| eval/normalized_episode_reward_std | 0.652 | +| loss/actor | -398 | +| loss/alpha | -0.00525 | +| loss/critic | 57.5 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.57 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.4816 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4735 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4899 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 966 | +| eval/episode_length_std | 120 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 16 | +| loss/actor | -394 | +| loss/alpha | 0.0142 | +| loss/critic | 59.5 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.62 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4799 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5048 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.561 | +| loss/actor | -394 | +| loss/alpha | -0.014 | +| loss/critic | 57.4 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.57 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5273 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4805 +num rollout transitions: 50000, reward mean: 4.4794 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5106 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.647 | +| loss/actor | -397 | +| loss/alpha | 0.00251 | +| loss/critic | 58.4 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.6 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4828 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4768 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4828 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4799 +num rollout transitions: 50000, reward mean: 4.4918 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 2.21 | +| loss/actor | -394 | +| loss/alpha | 0.00397 | +| loss/critic | 58 | +| penalty/lcb_mean | 1.56 | +| penalty/lcb_std | 2.6 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5289 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4857 +----------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 2.68 | +| loss/actor | -392 | +| loss/alpha | -0.000299 | +| loss/critic | 60.2 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.62 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2500000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4769 +num rollout transitions: 50000, reward mean: 4.4777 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5051 +---------------------------------------------------------------------------------- +| alpha | 0.118 | +| eval/episode_length | 968 | +| eval/episode_length_std | 106 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 13.2 | +| loss/actor | -394 | +| loss/alpha | -0.0129 | +| loss/critic | 60.9 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4789 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4745 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.85 | +| loss/actor | -394 | +| loss/alpha | 0.00315 | +| loss/critic | 62.3 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.67 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4816 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5019 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.869 | +| loss/actor | -392 | +| loss/alpha | 0.00214 | +| loss/critic | 63.9 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.69 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4824 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5031 +---------------------------------------------------------------------------------- +| alpha | 0.11 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 0.524 | +| loss/actor | -396 | +| loss/alpha | -0.00577 | +| loss/critic | 60.1 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.66 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5191 +---------------------------------------------------------------------------------- +| alpha | 0.11 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.672 | +| loss/actor | -396 | +| loss/alpha | 0.00188 | +| loss/critic | 58.9 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5320 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4992 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.818 | +| loss/actor | -393 | +| loss/alpha | 0.013 | +| loss/critic | 61.1 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.67 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5122 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 0.955 | +| loss/actor | -392 | +| loss/alpha | 0.0219 | +| loss/critic | 65.5 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.74 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4961 +---------------------------------------------------------------------------------- +| alpha | 0.113 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 0.708 | +| loss/actor | -395 | +| loss/alpha | -0.0294 | +| loss/critic | 64.2 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.74 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5226 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5120 +---------------------------------------------------------------------------------- +| alpha | 0.108 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 119 | +| eval/normalized_episode_reward_std | 0.884 | +| loss/actor | -393 | +| loss/alpha | -0.029 | +| loss/critic | 64.6 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.75 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4992 +---------------------------------------------------------------------------------- +| alpha | 0.113 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 119 | +| eval/normalized_episode_reward_std | 0.784 | +| loss/actor | -391 | +| loss/alpha | 0.0352 | +| loss/critic | 66.8 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.81 | +| rollout_info/dynamics_penalty_mean | 0 | +| rollout_info/dynamics_penalty_std | 0 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 49829.10s diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_456×tamp_26-0505-202557/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/kde/seed_456×tamp_26-0505-202557/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_456×tamp_26-0505-202557/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/kde/seed_456×tamp_26-0505-202557/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..5aac01d13cab0a81076fe666c0aba6f465948fb0 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/kde/seed_456×tamp_26-0505-202557/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 456, "device": "cuda:6", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_456×tamp_26-0112-030559/model", "reward_penalty_coef": 0.3, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/kde", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "kde", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "kde"} diff --git a/walker2d_medium_expert_sparse/mobile_based/kde/seed_456×tamp_26-0505-202557/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/kde/seed_456×tamp_26-0505-202557/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..74ccf958d0f388761020ddbe8c34d2589d358199 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/kde/seed_456×tamp_26-0505-202557/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.26302074094548433,113.6,8.339064695755754,-0.3699054581445653,0.11595688001063736,-296.9113988189024,-1.6541353361572593,20.98880853631143,1.0028355152728137,1.2714189146485229,0.0,0.0,50000.0,4.5257717488318425,50000 +0.13270357054769968,142.89,116.60196353406748,8.862656539269945,10.295022476072551,-433.7659058184781,-0.003481926126330871,26.48538063144669,1.0381531951904404,1.5451870814991013,0.0,0.0,50000.0,4.507994103675944,100000 +0.11950824266180525,127.23,27.666533935424578,8.05586241252008,3.253722875818045,-434.9972657592799,0.01290997781459239,25.238869391193386,1.0628685341489472,1.593465230418449,0.0,0.0,50000.0,4.5071140697069465,150000 +0.13981627233207183,127.1,51.90134873006673,5.294574806971551,3.5795746989561983,-430.2781313824485,0.023966364183835943,29.0332732966996,1.165398727934344,1.622861844859128,0.0,0.0,50000.0,4.50390199499879,200000 +0.1458308376643067,491.49,334.2566228214484,36.757782108035,33.18675495589872,-417.89111899047805,-0.005934173034653407,30.6621953582955,1.1686595085775757,1.6606436494516976,0.0,0.0,50000.0,4.508365998738585,250000 +0.14031497635602885,797.23,360.9043046293574,74.00724740640145,36.984110577373826,-413.98950567809976,-0.017057949647866392,31.74324454509713,1.1700844200408462,1.7355706246924316,0.0,0.0,50000.0,4.509813846113861,300000 +0.1377388135907047,1000.0,0.0,98.58727191371297,1.7654375544969525,-410.25323506408796,0.00827147855162644,32.750137897834456,1.1775382011592566,1.7701050595200052,0.0,0.0,50000.0,4.505191151419231,350000 +0.144155661494431,999.52,4.775939698111775,108.5341116480485,1.882246896611459,-406.4413557031327,0.00026873023089010305,35.26277013547969,1.2231978736293052,1.8266773400080216,0.0,0.0,50000.0,4.511803951820683,400000 +0.14265103183001257,1000.0,0.0,97.290885004722,2.1188863735912453,-404.4462819061303,0.005202277362495704,37.09403856512088,1.2531847980547106,1.9005471901714615,0.0,0.0,50000.0,4.507372797427194,450000 +0.14604439912706604,1000.0,0.0,105.97015320313146,2.162692297945646,-398.73124606750605,-0.005041203986853354,40.113292854022184,1.3075221419262877,2.0199562262344237,0.0,0.0,50000.0,4.504089979240215,500000 +0.15250721890091823,1000.0,0.0,100.49961850796798,1.4087393748731463,-396.36673011047327,0.0063634278682615526,40.271502928199844,1.3200114623320123,2.0128963248216727,0.0,0.0,50000.0,4.509976949591773,550000 +0.15489281806110836,1000.0,0.0,109.45091068063721,1.640435123018913,-392.7539054455609,0.009343933075293763,43.79786179691391,1.3796417844903603,2.1051466215133887,0.0,0.0,50000.0,4.505452290770132,600000 +0.14543461182445025,1000.0,0.0,101.51003749856,0.49436428427761375,-394.3808915331999,-0.016943137086853674,41.337485743159725,1.3160618892633782,2.0594040983831894,0.0,0.0,50000.0,4.502095547794558,650000 +0.14162553012520157,1000.0,0.0,106.66798188894869,0.5458339460188599,-391.6263056213437,-0.005251432843357491,42.3982702955058,1.3162163814663845,2.0888540105581237,0.0,0.0,50000.0,4.497742532170104,700000 +0.13891346915856237,1000.0,0.0,104.39225261226011,2.46926285442139,-391.2331083398413,0.013018142934516072,39.98215742679658,1.2989519280982018,2.018997885649205,0.0,0.0,50000.0,4.500456872098389,750000 +0.13915257258921843,1000.0,0.0,113.11067590616415,1.2659823147042502,-394.53158653198443,-0.010234146244786583,40.85249177154541,1.3195158821213357,2.067364362685671,0.0,0.0,50000.0,4.498917440298348,800000 +0.13643844957798515,1000.0,0.0,109.8150283727847,0.8458615717797223,-395.5315152392615,0.0034594051039963892,40.40244168884238,1.3213549347019375,2.0708084598124183,0.0,0.0,50000.0,4.504351417045718,850000 +0.13725826939612357,996.94,30.446615575462566,41.57516832560948,31.408293442240748,-393.58311552245993,-0.004255262393541632,42.79252330703729,1.363810519577237,2.1338087155902445,0.0,0.0,50000.0,4.501115501657237,900000 +0.13493302808851015,1000.0,0.0,108.69062887356071,2.3798660204810576,-396.4149641503885,0.000860128226131261,43.032442949771315,1.36585364524246,2.160783557567612,0.0,0.0,50000.0,4.498268617973371,950000 +0.13408431083530212,1000.0,0.0,110.79329316827278,1.3703714404963514,-395.04174718322855,0.002646005792506022,42.10916314300545,1.336115995806452,2.109990197412968,0.0,0.0,50000.0,4.5015852751487575,1000000 +0.12946500306278572,1000.0,0.0,110.80932132829298,1.683014958629369,-393.69680923645296,-0.007242108612023388,41.20773956680188,1.311925794757619,2.1018128700160585,0.0,0.0,50000.0,4.502548679587455,1050000 +0.1278793888616581,1000.0,0.0,109.4020389667246,0.38415592195467635,-394.82725938354264,0.0038510292394086172,41.50671687196791,1.3323756073653814,2.1084683973205456,0.0,0.0,50000.0,4.498184086285305,1100000 +0.12971780437499164,1000.0,0.0,114.65495255542591,1.1218765000365656,-399.413408911743,0.005014343002513036,42.105931007328046,1.3391306753217955,2.1188313761222,0.0,0.0,50000.0,4.499849060112984,1150000 +0.13341775205046083,915.47,162.18960848340438,100.57330762386805,18.96736263100091,-399.99768281615997,-0.0008178379140421076,43.88382514196385,1.3713291920924096,2.1745378665077855,0.0,0.0,50000.0,4.499006680462229,1200000 +0.13075632351487776,1000.0,0.0,108.19483204254816,0.826524334746624,-398.42672872192026,-0.0031074134957790513,44.95587571552333,1.3664362448823355,2.1611407994592486,0.0,0.0,50000.0,4.498731286528622,1250000 +0.128170720721931,1000.0,0.0,112.88803695730161,0.5907483428953834,-394.3543942724576,0.007255163841880834,43.928360629826145,1.3607981378292857,2.157930073853707,0.0,0.0,50000.0,4.499314099646531,1300000 +0.12947650010690204,1000.0,0.0,107.60123229066916,2.3840212463124244,-394.91535055419274,-0.005006237185187743,46.166773430119086,1.4056571036350478,2.240623289352676,0.0,0.0,50000.0,4.4973796379246,1350000 +0.13071581626832557,1000.0,0.0,115.0841286694513,0.27363400606953964,-395.2758786230456,-0.004919971300810641,48.43170666322615,1.4157521785223444,2.3087942973554365,0.0,0.0,50000.0,4.49701993582785,1400000 +0.12621569143489053,1000.0,0.0,112.81579597624417,0.5166191502455654,-395.4851102099642,0.002944004303403281,46.34572194791796,1.4009743090248052,2.25224752544283,0.0,0.0,50000.0,4.500210046968335,1450000 +0.12906094559252046,1000.0,0.0,111.75592405836451,0.4871183089818375,-394.37405166197925,-0.0033077118093147377,47.49932269762035,1.4215985904669852,2.2621698894560547,0.0,0.0,50000.0,4.495188528762246,1500000 +0.12944366948574929,1000.0,0.0,110.36533187479914,0.38459874781060993,-393.35627772094216,0.0036631393755228473,50.33589661792761,1.4622964306688302,2.3145081367909905,0.0,0.0,50000.0,4.497840293609048,1550000 +0.12791983864620068,1000.0,0.0,113.80918206983803,0.3225003397921985,-393.65002249145624,-0.006224959131423331,50.50264409463916,1.4593418352532228,2.3366807200419917,0.0,0.0,50000.0,4.499130830598016,1600000 +0.12910925143927449,1000.0,0.0,112.45427779030143,0.4523015232266673,-394.1948451049819,0.006498063537553161,54.30887322778703,1.4973184459340556,2.43794680345056,0.0,0.0,50000.0,4.50574134844806,1650000 +0.12895626357748718,1000.0,0.0,115.05934620160868,0.5702137211610758,-392.3214777526814,0.0018248807606101307,52.16864965139344,1.4671997643292152,2.399558901293291,0.0,0.0,50000.0,4.504451690092991,1700000 +0.13086340711444627,1000.0,0.0,115.08525516293562,0.8519750742498114,-392.96244382873874,-0.007845977625846826,52.59636194492325,1.5021146077322969,2.3855586466383523,0.0,0.0,50000.0,4.5006598560432405,1750000 +0.12582947591662166,1000.0,0.0,116.08720959593535,0.6523255072522193,-395.6855034082018,0.005242223158329676,52.1416212623973,1.4728581227362088,2.385739536294941,0.0,0.0,50000.0,4.4996053689206565,1800000 +0.1267408716347813,1000.0,0.0,116.67376910539406,0.47031117833572056,-398.9416131622328,-0.004603188376389434,51.28819484941472,1.4626784619378994,2.384722717108735,0.0,0.0,50000.0,4.504459533967771,1850000 +0.12498921055480915,1000.0,0.0,113.35298101796505,0.8271067646022655,-399.496527844848,-0.0019695844725893443,50.485147866839846,1.443755460268257,2.360877430930146,0.0,0.0,50000.0,4.501482473651391,1900000 +0.12179861283540785,996.02,26.278881254726205,116.16526445364057,3.4363600794367786,-399.16749680724837,-0.0006289923665672106,48.72216125070513,1.4424860648203008,2.3164995393669883,0.0,0.0,50000.0,4.498308584164216,1950000 +0.12783624289706638,956.58,148.38788225458305,106.65685145076695,18.40544946602892,-396.35639091308764,0.017405474813841273,52.34886941852595,1.488063217407455,2.3899308169901414,0.0,0.0,50000.0,4.500435215965389,2000000 +0.13225913122758362,1000.0,0.0,113.24372482910302,0.6500715294254835,-396.2589624646021,-0.012592803568281314,55.89515544584323,1.5423794466543221,2.488388481855398,0.0,0.0,50000.0,4.5002732368372165,2050000 +0.12266398408278852,1000.0,0.0,113.45564156034817,1.4330384120690134,-397.917727588502,0.0021074752266705085,53.06235012853653,1.5048108618855425,2.4405142347228708,0.0,0.0,50000.0,4.502507834287615,2100000 +0.12463345014482843,1000.0,0.0,113.50933942780823,0.506613852351447,-397.1326482122814,6.302644699817184e-05,54.49544554271795,1.5195876031088875,2.492976108564144,0.0,0.0,50000.0,4.502704002420984,2150000 +0.12107010281458518,1000.0,0.0,113.72609176698498,0.5386348509034506,-398.7294838659725,0.002494178003668687,53.330109590645115,1.4957209714710729,2.461194330850824,0.0,0.0,50000.0,4.50156512752417,2200000 +0.12513508392125156,1000.0,0.0,118.86396913436685,0.6524084673319872,-397.83161136353027,-0.005246816434264192,57.521250461997745,1.5483538516211337,2.565871728596649,0.0,0.0,50000.0,4.501557746051722,2250000 +0.12811278108462582,966.13,120.41317660455603,109.62310491008739,15.954479622957413,-394.1087990539585,0.014246181338392387,59.534203106976065,1.5844628039908495,2.6179337510204252,0.0,0.0,50000.0,4.501533345244104,2300000 +0.12003827814042535,1000.0,0.0,113.98181329015009,0.5612327162667415,-393.81927542724685,-0.014021206624656874,57.42405031055365,1.531441270563597,2.566685013097522,0.0,0.0,50000.0,4.50204415447669,2350000 +0.12281747420117275,1000.0,0.0,112.21154168369327,0.6469335862843495,-396.5087057104528,0.00250674491506063,58.364795175781,1.5432501530361142,2.595656158485408,0.0,0.0,50000.0,4.499458407002928,2400000 +0.11625090010970748,1000.0,0.0,117.3120291001493,2.2142324675706364,-394.0257401415986,0.003973587187565809,58.02066089559555,1.564032684016215,2.5967360218238746,0.0,0.0,50000.0,4.496967383038151,2450000 +0.12476399625942039,1000.0,0.0,105.40595929157024,2.6813006906952976,-392.32415939208994,-0.00029867371898141353,60.24803984802236,1.577691328752035,2.6195781257152437,0.0,0.0,50000.0,4.502290796042481,2500000 +0.1177295739087449,967.59,105.78318344614136,111.93928122016878,13.242489888677383,-393.5703561181625,-0.012892397700883448,60.89682545650491,1.5889857692825837,2.6495805607891483,0.0,0.0,50000.0,4.5015140292912355,2550000 +0.12062600902706215,1000.0,0.0,115.58421710685187,0.8498314020767314,-393.82575504883187,0.0031528581247848957,62.310319918232025,1.606857481458205,2.6738208908986776,0.0,0.0,50000.0,4.503167754752498,2600000 +0.12117799428194674,1000.0,0.0,116.17033860113182,0.8690703657738283,-391.83703060791174,0.0021370313796028125,63.939768017405854,1.6372984623884994,2.6858047833633325,0.0,0.0,50000.0,4.498721136011356,2650000 +0.11035762422382797,1000.0,0.0,118.44407483254014,0.5239482732278278,-395.9761223785419,-0.005766843334063921,60.12276716453551,1.5902230672848374,2.6583727641749237,0.0,0.0,50000.0,4.503261877974726,2700000 +0.11040832270622183,1000.0,0.0,117.05130689410548,0.6717537743839466,-395.64628590088313,0.0018825626332313794,58.89048119346661,1.5512369387638518,2.6284232037043553,0.0,0.0,50000.0,4.504898354218937,2750000 +0.11619152972966434,1000.0,0.0,116.0448404746151,0.818340074480549,-393.3086895532276,0.012988059868253944,61.07027225927355,1.5863112381744247,2.6726270572233304,0.0,0.0,50000.0,4.503917051769528,2800000 +0.11960641937568874,1000.0,0.0,118.49350661346291,0.955132971899396,-392.00360151672174,0.021876733452416783,65.49159657449692,1.6545735853648387,2.7393936028790455,0.0,0.0,50000.0,4.505705606589957,2850000 +0.1129537836465261,1000.0,0.0,117.56895000698906,0.7077528957325597,-394.527966557006,-0.029439682675451478,64.18341192903489,1.6229628145813957,2.7431123861479705,0.0,0.0,50000.0,4.509036305645187,2900000 +0.10827912305459308,1000.0,0.0,118.98211250669655,0.884147506805082,-393.04926341003403,-0.029043680929094585,64.58864446525537,1.6344589481663687,2.746119897267746,0.0,0.0,50000.0,4.50899899507531,2950000 +0.11322722212016531,1000.0,0.0,118.53993341340883,0.783540988527869,-391.1350035864251,0.0351901075275984,66.75271771539747,1.6694847181534758,2.812881079289885,0.0,0.0,50000.0,4.503797759724416,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193338/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193338/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..3b4d61db83ee3f3804e1be3ef692df20849c0fbe --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193338/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 5000, reward mean: 4.4344 +num rollout transitions: 5000, reward mean: 4.4693 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5480 +num rollout transitions: 5000, reward mean: 4.5338 +num rollout transitions: 5000, reward mean: 4.4750 +num rollout transitions: 5000, reward mean: 4.4440 +num rollout transitions: 5000, reward mean: 4.4687 +num rollout transitions: 5000, reward mean: 4.4884 +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.5278 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5330 +num rollout transitions: 5000, reward mean: 4.5548 +num rollout transitions: 5000, reward mean: 4.5458 +num rollout transitions: 5000, reward mean: 4.5194 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.4725 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.5323 +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.4445 +num rollout transitions: 5000, reward mean: 4.3534 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.4618 +num rollout transitions: 5000, reward mean: 4.5709 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5301 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.4805 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.4687 +num rollout transitions: 5000, reward mean: 4.5383 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.5582 +num rollout transitions: 5000, reward mean: 4.5525 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5371 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.5101 +---------------------------------------------------------------------------------- +| alpha | 0.272 | +| eval/episode_length | 153 | +| eval/episode_length_std | 38.8 | +| eval/normalized_episode_reward | 7.04 | +| eval/normalized_episode_reward_std | 3.92 | +| loss/actor | -315 | +| loss/alpha | -1.66 | +| loss/critic | 27.6 | +| penalty/lcb_mean | 1.17 | +| penalty/lcb_std | 1.61 | +| rollout_info/dynamics_penalty_mean | 0.0155 | +| rollout_info/dynamics_penalty_std | 0.0978 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4665 +num rollout transitions: 5000, reward mean: 4.5230 +num rollout transitions: 5000, reward mean: 4.5263 +num rollout transitions: 5000, reward mean: 4.5333 +num rollout transitions: 5000, reward mean: 4.4925 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.4686 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.2950 +num rollout transitions: 5000, reward mean: 4.3834 +num rollout transitions: 5000, reward mean: 4.3899 +num rollout transitions: 5000, reward mean: 4.4762 +num rollout transitions: 5000, reward mean: 4.5399 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.4808 +num rollout transitions: 5000, reward mean: 4.4798 +num rollout transitions: 5000, reward mean: 4.4826 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5258 +num rollout transitions: 5000, reward mean: 4.5670 +num rollout transitions: 5000, reward mean: 4.5444 +num rollout transitions: 5000, reward mean: 4.5274 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.4428 +num rollout transitions: 5000, reward mean: 4.4579 +num rollout transitions: 5000, reward mean: 4.4812 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4679 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.4609 +num rollout transitions: 5000, reward mean: 4.4543 +num rollout transitions: 5000, reward mean: 4.4523 +num rollout transitions: 5000, reward mean: 4.5379 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5271 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.4511 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5339 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.4734 +---------------------------------------------------------------------------------- +| alpha | 0.205 | +| eval/episode_length | 167 | +| eval/episode_length_std | 44.7 | +| eval/normalized_episode_reward | 7.18 | +| eval/normalized_episode_reward_std | 3.91 | +| loss/actor | -501 | +| loss/alpha | 0.246 | +| loss/critic | 61.1 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.5 | +| rollout_info/dynamics_penalty_mean | 0.00715 | +| rollout_info/dynamics_penalty_std | 0.0728 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4763 +num rollout transitions: 5000, reward mean: 4.5337 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.4770 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5536 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.5536 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.4571 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.4615 +num rollout transitions: 5000, reward mean: 4.5444 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.4764 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.4641 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.4880 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.4795 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.5071 +num rollout transitions: 5000, reward mean: 4.5115 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.4778 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4860 +---------------------------------------------------------------------------------- +| alpha | 0.173 | +| eval/episode_length | 32.3 | +| eval/episode_length_std | 5.56 | +| eval/normalized_episode_reward | 0.0644 | +| eval/normalized_episode_reward_std | 0.0579 | +| loss/actor | -436 | +| loss/alpha | -0.0197 | +| loss/critic | 41.5 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.13 | +| rollout_info/dynamics_penalty_mean | 0.00408 | +| rollout_info/dynamics_penalty_std | 0.0596 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4437 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.5306 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.4818 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.4688 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4721 +num rollout transitions: 5000, reward mean: 4.5568 +num rollout transitions: 5000, reward mean: 4.5323 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5177 +num rollout transitions: 5000, reward mean: 4.4655 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.5248 +num rollout transitions: 5000, reward mean: 4.5276 +num rollout transitions: 5000, reward mean: 4.4788 +num rollout transitions: 5000, reward mean: 4.5240 +num rollout transitions: 5000, reward mean: 4.4772 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5205 +num rollout transitions: 5000, reward mean: 4.4824 +num rollout transitions: 5000, reward mean: 4.5241 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.5290 +num rollout transitions: 5000, reward mean: 4.5315 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4776 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.4663 +num rollout transitions: 5000, reward mean: 4.5347 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.5338 +---------------------------------------------------------------------------------- +| alpha | 0.155 | +| eval/episode_length | 128 | +| eval/episode_length_std | 13.5 | +| eval/normalized_episode_reward | 7.98 | +| eval/normalized_episode_reward_std | 1.38 | +| loss/actor | -432 | +| loss/alpha | 0.0237 | +| loss/critic | 31.5 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.97 | +| rollout_info/dynamics_penalty_mean | 0.00568 | +| rollout_info/dynamics_penalty_std | 0.0715 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.5537 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.5330 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.5514 +num rollout transitions: 5000, reward mean: 4.5252 +num rollout transitions: 5000, reward mean: 4.5115 +num rollout transitions: 5000, reward mean: 4.5320 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.5168 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.5376 +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.5438 +num rollout transitions: 5000, reward mean: 4.5237 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5364 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.5250 +num rollout transitions: 5000, reward mean: 4.4789 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.5381 +num rollout transitions: 5000, reward mean: 4.5441 +num rollout transitions: 5000, reward mean: 4.5288 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.4773 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.5344 +num rollout transitions: 5000, reward mean: 4.5455 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.5263 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5270 +---------------------------------------------------------------------------------- +| alpha | 0.172 | +| eval/episode_length | 168 | +| eval/episode_length_std | 14.3 | +| eval/normalized_episode_reward | 9.09 | +| eval/normalized_episode_reward_std | 0.961 | +| loss/actor | -423 | +| loss/alpha | -0.00386 | +| loss/critic | 38.1 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 1.99 | +| rollout_info/dynamics_penalty_mean | 0.00494 | +| rollout_info/dynamics_penalty_std | 0.0662 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.4674 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.4817 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.5286 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.4801 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.4712 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5349 +num rollout transitions: 5000, reward mean: 4.5352 +num rollout transitions: 5000, reward mean: 4.5307 +num rollout transitions: 5000, reward mean: 4.5128 +num rollout transitions: 5000, reward mean: 4.5402 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.5454 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.4846 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.5260 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.4973 +---------------------------------------------------------------------------------- +| alpha | 0.154 | +| eval/episode_length | 155 | +| eval/episode_length_std | 54 | +| eval/normalized_episode_reward | 7.54 | +| eval/normalized_episode_reward_std | 3.36 | +| loss/actor | -419 | +| loss/alpha | 0.00786 | +| loss/critic | 35.6 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 1.96 | +| rollout_info/dynamics_penalty_mean | 0.0041 | +| rollout_info/dynamics_penalty_std | 0.0591 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4885 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.5175 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.4808 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.4797 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.5453 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.4823 +num rollout transitions: 5000, reward mean: 4.5065 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.5263 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.5570 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.5526 +num rollout transitions: 5000, reward mean: 4.5395 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.5460 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.4664 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5154 +num rollout transitions: 5000, reward mean: 4.4730 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.4619 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.5298 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.5165 +---------------------------------------------------------------------------------- +| alpha | 0.168 | +| eval/episode_length | 192 | +| eval/episode_length_std | 24.6 | +| eval/normalized_episode_reward | 11.1 | +| eval/normalized_episode_reward_std | 2.28 | +| loss/actor | -429 | +| loss/alpha | -0.00349 | +| loss/critic | 40.4 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.16 | +| rollout_info/dynamics_penalty_mean | 0.0041 | +| rollout_info/dynamics_penalty_std | 0.0594 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.5497 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4699 +num rollout transitions: 5000, reward mean: 4.4641 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.4865 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.4738 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.5313 +num rollout transitions: 5000, reward mean: 4.5306 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5369 +num rollout transitions: 5000, reward mean: 4.5329 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5477 +num rollout transitions: 5000, reward mean: 4.4719 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.4497 +num rollout transitions: 5000, reward mean: 4.4711 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5115 +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5255 +num rollout transitions: 5000, reward mean: 4.5191 +---------------------------------------------------------------------------------- +| alpha | 0.176 | +| eval/episode_length | 185 | +| eval/episode_length_std | 10.9 | +| eval/normalized_episode_reward | 11.9 | +| eval/normalized_episode_reward_std | 1.18 | +| loss/actor | -411 | +| loss/alpha | 0.018 | +| loss/critic | 40.2 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.08 | +| rollout_info/dynamics_penalty_mean | 0.0027 | +| rollout_info/dynamics_penalty_std | 0.0487 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5534 +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.5295 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.5217 +num rollout transitions: 5000, reward mean: 4.5257 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.5341 +num rollout transitions: 5000, reward mean: 4.5814 +num rollout transitions: 5000, reward mean: 4.5308 +num rollout transitions: 5000, reward mean: 4.5071 +num rollout transitions: 5000, reward mean: 4.4786 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.4817 +num rollout transitions: 5000, reward mean: 4.4824 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.5311 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.4713 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.5394 +num rollout transitions: 5000, reward mean: 4.5329 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.4684 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5382 +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.5188 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.5855 +num rollout transitions: 5000, reward mean: 4.5399 +num rollout transitions: 5000, reward mean: 4.5537 +num rollout transitions: 5000, reward mean: 4.5493 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.5439 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5363 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.5033 +---------------------------------------------------------------------------------- +| alpha | 0.185 | +| eval/episode_length | 268 | +| eval/episode_length_std | 202 | +| eval/normalized_episode_reward | 15.2 | +| eval/normalized_episode_reward_std | 21.6 | +| loss/actor | -392 | +| loss/alpha | -0.0173 | +| loss/critic | 41.3 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.06 | +| rollout_info/dynamics_penalty_mean | 0.00324 | +| rollout_info/dynamics_penalty_std | 0.0534 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5373 +num rollout transitions: 5000, reward mean: 4.5290 +num rollout transitions: 5000, reward mean: 4.5019 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.5157 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.5343 +num rollout transitions: 5000, reward mean: 4.5364 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.5400 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.5435 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.5490 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5256 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.5512 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.5270 +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.5192 +num rollout transitions: 5000, reward mean: 4.4797 +num rollout transitions: 5000, reward mean: 4.5106 +num rollout transitions: 5000, reward mean: 4.5312 +---------------------------------------------------------------------------------- +| alpha | 0.176 | +| eval/episode_length | 213 | +| eval/episode_length_std | 11.7 | +| eval/normalized_episode_reward | 14.7 | +| eval/normalized_episode_reward_std | 1.1 | +| loss/actor | -385 | +| loss/alpha | -0.00849 | +| loss/critic | 39 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 1.97 | +| rollout_info/dynamics_penalty_mean | 0.00339 | +| rollout_info/dynamics_penalty_std | 0.0556 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.5258 +num rollout transitions: 5000, reward mean: 4.4630 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.5273 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.4565 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.4878 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.5417 +num rollout transitions: 5000, reward mean: 4.5343 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5295 +num rollout transitions: 5000, reward mean: 4.5284 +num rollout transitions: 5000, reward mean: 4.5288 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.4627 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5013 +num rollout transitions: 5000, reward mean: 4.4860 +num rollout transitions: 5000, reward mean: 4.4872 +num rollout transitions: 5000, reward mean: 4.5356 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.5174 +num rollout transitions: 5000, reward mean: 4.4848 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.4846 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.5004 +---------------------------------------------------------------------------------- +| alpha | 0.172 | +| eval/episode_length | 150 | +| eval/episode_length_std | 2.09 | +| eval/normalized_episode_reward | 9.25 | +| eval/normalized_episode_reward_std | 0.0745 | +| loss/actor | -389 | +| loss/alpha | -0.00821 | +| loss/critic | 39.5 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.05 | +| rollout_info/dynamics_penalty_mean | 0.00406 | +| rollout_info/dynamics_penalty_std | 0.0613 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4988 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.4955 +num rollout transitions: 5000, reward mean: 4.4639 +num rollout transitions: 5000, reward mean: 4.5228 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.4757 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.4902 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.4702 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5381 +num rollout transitions: 5000, reward mean: 4.5283 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.5494 +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.4730 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.5338 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.4762 +num rollout transitions: 5000, reward mean: 4.5194 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5319 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.5407 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.5095 +---------------------------------------------------------------------------------- +| alpha | 0.176 | +| eval/episode_length | 111 | +| eval/episode_length_std | 5.24 | +| eval/normalized_episode_reward | 5.25 | +| eval/normalized_episode_reward_std | 0.148 | +| loss/actor | -393 | +| loss/alpha | 0.0234 | +| loss/critic | 42.7 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.14 | +| rollout_info/dynamics_penalty_mean | 0.00351 | +| rollout_info/dynamics_penalty_std | 0.0567 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5252 +num rollout transitions: 5000, reward mean: 4.5263 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5345 +num rollout transitions: 5000, reward mean: 4.5255 +num rollout transitions: 5000, reward mean: 4.5720 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5374 +num rollout transitions: 5000, reward mean: 4.5228 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5266 +num rollout transitions: 5000, reward mean: 4.5450 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.5576 +num rollout transitions: 5000, reward mean: 4.5470 +num rollout transitions: 5000, reward mean: 4.5567 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.5468 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.4815 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.5435 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.5377 +num rollout transitions: 5000, reward mean: 4.5766 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.5257 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5052 +---------------------------------------------------------------------------------- +| alpha | 0.197 | +| eval/episode_length | 488 | +| eval/episode_length_std | 97 | +| eval/normalized_episode_reward | 43.6 | +| eval/normalized_episode_reward_std | 11.4 | +| loss/actor | -387 | +| loss/alpha | 0.00675 | +| loss/critic | 50.2 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.19 | +| rollout_info/dynamics_penalty_mean | 0.00334 | +| rollout_info/dynamics_penalty_std | 0.0553 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.5227 +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5410 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.5305 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.5216 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.5294 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.5072 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.4896 +num rollout transitions: 5000, reward mean: 4.5180 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.5170 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.4865 +num rollout transitions: 5000, reward mean: 4.4824 +num rollout transitions: 5000, reward mean: 4.5441 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.5305 +num rollout transitions: 5000, reward mean: 4.4794 +---------------------------------------------------------------------------------- +| alpha | 0.194 | +| eval/episode_length | 574 | +| eval/episode_length_std | 260 | +| eval/normalized_episode_reward | 54.9 | +| eval/normalized_episode_reward_std | 28.3 | +| loss/actor | -377 | +| loss/alpha | 0.00531 | +| loss/critic | 48.1 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.19 | +| rollout_info/dynamics_penalty_mean | 0.00312 | +| rollout_info/dynamics_penalty_std | 0.0534 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5192 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.5393 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.4880 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.4872 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.4770 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.5314 +num rollout transitions: 5000, reward mean: 4.4833 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.4797 +num rollout transitions: 5000, reward mean: 4.4826 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.4588 +num rollout transitions: 5000, reward mean: 4.4750 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5315 +num rollout transitions: 5000, reward mean: 4.5220 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.4851 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5301 +num rollout transitions: 5000, reward mean: 4.5205 +num rollout transitions: 5000, reward mean: 4.5001 +---------------------------------------------------------------------------------- +| alpha | 0.194 | +| eval/episode_length | 106 | +| eval/episode_length_std | 1.66 | +| eval/normalized_episode_reward | 4.82 | +| eval/normalized_episode_reward_std | 0.0564 | +| loss/actor | -362 | +| loss/alpha | -0.00457 | +| loss/critic | 50.8 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 0.00337 | +| rollout_info/dynamics_penalty_std | 0.0558 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.4673 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.5397 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.5341 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5286 +num rollout transitions: 5000, reward mean: 4.5400 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.5387 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5290 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.5358 +num rollout transitions: 5000, reward mean: 4.5325 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.4765 +num rollout transitions: 5000, reward mean: 4.4803 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.5316 +num rollout transitions: 5000, reward mean: 4.5197 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.5330 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.5278 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.5783 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5406 +num rollout transitions: 5000, reward mean: 4.5308 +num rollout transitions: 5000, reward mean: 4.5128 +num rollout transitions: 5000, reward mean: 4.5529 +num rollout transitions: 5000, reward mean: 4.4772 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5240 +----------------------------------------------------------------------------------- +| alpha | 0.195 | +| eval/episode_length | 963 | +| eval/episode_length_std | 131 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 15.4 | +| loss/actor | -361 | +| loss/alpha | -0.000806 | +| loss/critic | 51.9 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.23 | +| rollout_info/dynamics_penalty_mean | 0.00321 | +| rollout_info/dynamics_penalty_std | 0.0543 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 800000 | +----------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5351 +num rollout transitions: 5000, reward mean: 4.5291 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.5493 +num rollout transitions: 5000, reward mean: 4.5437 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.5302 +num rollout transitions: 5000, reward mean: 4.5396 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.4879 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.5380 +num rollout transitions: 5000, reward mean: 4.5248 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5347 +num rollout transitions: 5000, reward mean: 4.4561 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.5393 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.4773 +num rollout transitions: 5000, reward mean: 4.5013 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.5319 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.5360 +num rollout transitions: 5000, reward mean: 4.5373 +num rollout transitions: 5000, reward mean: 4.5181 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.4974 +---------------------------------------------------------------------------------- +| alpha | 0.186 | +| eval/episode_length | 102 | +| eval/episode_length_std | 2.7 | +| eval/normalized_episode_reward | 4.57 | +| eval/normalized_episode_reward_std | 0.0968 | +| loss/actor | -368 | +| loss/alpha | -0.0074 | +| loss/critic | 51.7 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.27 | +| rollout_info/dynamics_penalty_mean | 0.00268 | +| rollout_info/dynamics_penalty_std | 0.0491 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4455 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.5180 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.4502 +num rollout transitions: 5000, reward mean: 4.5292 +num rollout transitions: 5000, reward mean: 4.5280 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.5178 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.5225 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.4676 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.5245 +num rollout transitions: 5000, reward mean: 4.5379 +num rollout transitions: 5000, reward mean: 4.5338 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.5253 +num rollout transitions: 5000, reward mean: 4.5384 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.5249 +num rollout transitions: 5000, reward mean: 4.5313 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.4655 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.4764 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.4879 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.5204 +---------------------------------------------------------------------------------- +| alpha | 0.182 | +| eval/episode_length | 824 | +| eval/episode_length_std | 308 | +| eval/normalized_episode_reward | 86.1 | +| eval/normalized_episode_reward_std | 35.5 | +| loss/actor | -362 | +| loss/alpha | -0.00259 | +| loss/critic | 51.9 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 0.00212 | +| rollout_info/dynamics_penalty_std | 0.0437 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.5175 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.4730 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.5162 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.5227 +num rollout transitions: 5000, reward mean: 4.4672 +num rollout transitions: 5000, reward mean: 4.5011 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.4851 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.5555 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.4735 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5322 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5305 +num rollout transitions: 5000, reward mean: 4.5283 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5280 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.5434 +num rollout transitions: 5000, reward mean: 4.5293 +num rollout transitions: 5000, reward mean: 4.5011 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.4783 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.5271 +num rollout transitions: 5000, reward mean: 4.4921 +---------------------------------------------------------------------------------- +| alpha | 0.195 | +| eval/episode_length | 161 | +| eval/episode_length_std | 13 | +| eval/normalized_episode_reward | 8.44 | +| eval/normalized_episode_reward_std | 1.2 | +| loss/actor | -360 | +| loss/alpha | 0.00722 | +| loss/critic | 53.8 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.3 | +| rollout_info/dynamics_penalty_mean | 0.00259 | +| rollout_info/dynamics_penalty_std | 0.0489 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4744 +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.4662 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.4730 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5293 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5259 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.5462 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.5071 +num rollout transitions: 5000, reward mean: 4.4885 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.4860 +num rollout transitions: 5000, reward mean: 4.4988 +num rollout transitions: 5000, reward mean: 4.5416 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5194 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5154 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.5330 +num rollout transitions: 5000, reward mean: 4.5450 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.5283 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.5557 +num rollout transitions: 5000, reward mean: 4.5399 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.5361 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.4685 +num rollout transitions: 5000, reward mean: 4.5175 +num rollout transitions: 5000, reward mean: 4.5299 +---------------------------------------------------------------------------------- +| alpha | 0.188 | +| eval/episode_length | 172 | +| eval/episode_length_std | 7.47 | +| eval/normalized_episode_reward | 10.9 | +| eval/normalized_episode_reward_std | 0.672 | +| loss/actor | -359 | +| loss/alpha | 0.00167 | +| loss/critic | 54.4 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.36 | +| rollout_info/dynamics_penalty_mean | 0.00221 | +| rollout_info/dynamics_penalty_std | 0.0452 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5476 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5398 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.4747 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.5170 +num rollout transitions: 5000, reward mean: 4.5296 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.4756 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.4756 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.5368 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5351 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.4882 +num rollout transitions: 5000, reward mean: 4.5216 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.5503 +num rollout transitions: 5000, reward mean: 4.5408 +num rollout transitions: 5000, reward mean: 4.5532 +num rollout transitions: 5000, reward mean: 4.5589 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5402 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5235 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.4940 +num rollout transitions: 5000, reward mean: 4.5237 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.5315 +num rollout transitions: 5000, reward mean: 4.5319 +num rollout transitions: 5000, reward mean: 4.5429 +---------------------------------------------------------------------------------- +| alpha | 0.197 | +| eval/episode_length | 686 | +| eval/episode_length_std | 430 | +| eval/normalized_episode_reward | 71.6 | +| eval/normalized_episode_reward_std | 47.9 | +| loss/actor | -358 | +| loss/alpha | 0.00545 | +| loss/critic | 56.3 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.27 | +| rollout_info/dynamics_penalty_mean | 0.00289 | +| rollout_info/dynamics_penalty_std | 0.0513 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.5276 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.4761 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5193 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.5465 +num rollout transitions: 5000, reward mean: 4.5390 +num rollout transitions: 5000, reward mean: 4.5473 +num rollout transitions: 5000, reward mean: 4.5193 +num rollout transitions: 5000, reward mean: 4.4707 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4727 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.4787 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5318 +num rollout transitions: 5000, reward mean: 4.5393 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4711 +num rollout transitions: 5000, reward mean: 4.5271 +num rollout transitions: 5000, reward mean: 4.4687 +num rollout transitions: 5000, reward mean: 4.4744 +num rollout transitions: 5000, reward mean: 4.4660 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.4605 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.4479 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.5316 +num rollout transitions: 5000, reward mean: 4.5308 +num rollout transitions: 5000, reward mean: 4.5257 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.4852 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4852 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.5253 +---------------------------------------------------------------------------------- +| alpha | 0.189 | +| eval/episode_length | 921 | +| eval/episode_length_std | 168 | +| eval/normalized_episode_reward | 90.8 | +| eval/normalized_episode_reward_std | 17.7 | +| loss/actor | -362 | +| loss/alpha | -0.00499 | +| loss/critic | 59.8 | +| penalty/lcb_mean | 1.71 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0.00333 | +| rollout_info/dynamics_penalty_std | 0.0548 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.5460 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5320 +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5515 +num rollout transitions: 5000, reward mean: 4.5426 +num rollout transitions: 5000, reward mean: 4.5405 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5304 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.4819 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.5280 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5346 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.4708 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4723 +num rollout transitions: 5000, reward mean: 4.4563 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.4817 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.4741 +num rollout transitions: 5000, reward mean: 4.4804 +num rollout transitions: 5000, reward mean: 4.4513 +num rollout transitions: 5000, reward mean: 4.4840 +----------------------------------------------------------------------------------- +| alpha | 0.204 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 95.6 | +| eval/normalized_episode_reward_std | 0.768 | +| loss/actor | -353 | +| loss/alpha | -0.000963 | +| loss/critic | 68.9 | +| penalty/lcb_mean | 1.82 | +| penalty/lcb_std | 2.58 | +| rollout_info/dynamics_penalty_mean | 0.00355 | +| rollout_info/dynamics_penalty_std | 0.0574 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1150000 | +----------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4823 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.5194 +num rollout transitions: 5000, reward mean: 4.5309 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5260 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.5301 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.5197 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.5276 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.4885 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.5473 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.4724 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.4700 +num rollout transitions: 5000, reward mean: 4.4826 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.4923 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4783 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5175 +num rollout transitions: 5000, reward mean: 4.4945 +---------------------------------------------------------------------------------- +| alpha | 0.194 | +| eval/episode_length | 907 | +| eval/episode_length_std | 165 | +| eval/normalized_episode_reward | 92.8 | +| eval/normalized_episode_reward_std | 19.2 | +| loss/actor | -346 | +| loss/alpha | 0.00242 | +| loss/critic | 58.2 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 0.00368 | +| rollout_info/dynamics_penalty_std | 0.0584 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4820 +num rollout transitions: 5000, reward mean: 4.5017 +num rollout transitions: 5000, reward mean: 4.5451 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.4716 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.5265 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.4827 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.4601 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.4828 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.5387 +num rollout transitions: 5000, reward mean: 4.5340 +num rollout transitions: 5000, reward mean: 4.4753 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5346 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.4691 +num rollout transitions: 5000, reward mean: 4.4708 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.4925 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.4616 +num rollout transitions: 5000, reward mean: 4.5135 +---------------------------------------------------------------------------------- +| alpha | 0.191 | +| eval/episode_length | 166 | +| eval/episode_length_std | 8.59 | +| eval/normalized_episode_reward | 12.4 | +| eval/normalized_episode_reward_std | 0.95 | +| loss/actor | -340 | +| loss/alpha | -0.00431 | +| loss/critic | 59.1 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 0.0025 | +| rollout_info/dynamics_penalty_std | 0.0483 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4706 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5405 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5275 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.5250 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.5235 +num rollout transitions: 5000, reward mean: 4.4597 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.4815 +num rollout transitions: 5000, reward mean: 4.4731 +num rollout transitions: 5000, reward mean: 4.5390 +num rollout transitions: 5000, reward mean: 4.4896 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.4815 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.4582 +num rollout transitions: 5000, reward mean: 4.4823 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.5334 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.5288 +num rollout transitions: 5000, reward mean: 4.5505 +num rollout transitions: 5000, reward mean: 4.5306 +num rollout transitions: 5000, reward mean: 4.5264 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.4732 +num rollout transitions: 5000, reward mean: 4.4727 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5008 +---------------------------------------------------------------------------------- +| alpha | 0.21 | +| eval/episode_length | 144 | +| eval/episode_length_std | 11.2 | +| eval/normalized_episode_reward | 7.59 | +| eval/normalized_episode_reward_std | 0.599 | +| loss/actor | -345 | +| loss/alpha | 0.0129 | +| loss/critic | 68.7 | +| penalty/lcb_mean | 1.83 | +| penalty/lcb_std | 2.53 | +| rollout_info/dynamics_penalty_mean | 0.00346 | +| rollout_info/dynamics_penalty_std | 0.0547 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.5080 +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.5614 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.5175 +num rollout transitions: 5000, reward mean: 4.4885 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.5238 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.4734 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5287 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.4659 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.5228 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.5530 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.5300 +num rollout transitions: 5000, reward mean: 4.4779 +num rollout transitions: 5000, reward mean: 4.5305 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.4852 +num rollout transitions: 5000, reward mean: 4.4772 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4804 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.5517 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.5257 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.5357 +num rollout transitions: 5000, reward mean: 4.5035 +---------------------------------------------------------------------------------- +| alpha | 0.201 | +| eval/episode_length | 34 | +| eval/episode_length_std | 9.31 | +| eval/normalized_episode_reward | 0.149 | +| eval/normalized_episode_reward_std | 0.0878 | +| loss/actor | -333 | +| loss/alpha | -0.0024 | +| loss/critic | 62 | +| penalty/lcb_mean | 1.72 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0.00174 | +| rollout_info/dynamics_penalty_std | 0.0398 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.4656 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.4708 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.4740 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.4709 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.4774 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.4786 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.5299 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4747 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.5306 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.4691 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.4763 +num rollout transitions: 5000, reward mean: 4.4784 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.5299 +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.4739 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.5254 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.4752 +num rollout transitions: 5000, reward mean: 4.5080 +num rollout transitions: 5000, reward mean: 4.5286 +---------------------------------------------------------------------------------- +| alpha | 0.205 | +| eval/episode_length | 521 | +| eval/episode_length_std | 204 | +| eval/normalized_episode_reward | 48.7 | +| eval/normalized_episode_reward_std | 24.2 | +| loss/actor | -345 | +| loss/alpha | 0.00295 | +| loss/critic | 68.9 | +| penalty/lcb_mean | 1.84 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.00265 | +| rollout_info/dynamics_penalty_std | 0.0494 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5196 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.5187 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.5485 +num rollout transitions: 5000, reward mean: 4.5355 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5192 +num rollout transitions: 5000, reward mean: 4.4753 +num rollout transitions: 5000, reward mean: 4.4789 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5192 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.4839 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.5451 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5213 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.4930 +num rollout transitions: 5000, reward mean: 4.5384 +num rollout transitions: 5000, reward mean: 4.4782 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.4763 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.4938 +---------------------------------------------------------------------------------- +| alpha | 0.205 | +| eval/episode_length | 858 | +| eval/episode_length_std | 202 | +| eval/normalized_episode_reward | 89.9 | +| eval/normalized_episode_reward_std | 22.6 | +| loss/actor | -355 | +| loss/alpha | 0.00167 | +| loss/critic | 67.1 | +| penalty/lcb_mean | 1.83 | +| penalty/lcb_std | 2.53 | +| rollout_info/dynamics_penalty_mean | 0.00305 | +| rollout_info/dynamics_penalty_std | 0.0522 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.4963 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.4791 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.4753 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.5567 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.5344 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.4421 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.5162 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.4842 +num rollout transitions: 5000, reward mean: 4.4776 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.4570 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.5381 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.5294 +num rollout transitions: 5000, reward mean: 4.4777 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.4828 +num rollout transitions: 5000, reward mean: 4.4771 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.4913 +---------------------------------------------------------------------------------- +| alpha | 0.201 | +| eval/episode_length | 788 | +| eval/episode_length_std | 229 | +| eval/normalized_episode_reward | 79 | +| eval/normalized_episode_reward_std | 26.7 | +| loss/actor | -360 | +| loss/alpha | -0.00171 | +| loss/critic | 71 | +| penalty/lcb_mean | 1.91 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0.00229 | +| rollout_info/dynamics_penalty_std | 0.0455 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4822 +num rollout transitions: 5000, reward mean: 4.4683 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.5464 +num rollout transitions: 5000, reward mean: 4.5252 +num rollout transitions: 5000, reward mean: 4.4496 +num rollout transitions: 5000, reward mean: 4.4896 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.4949 +num rollout transitions: 5000, reward mean: 4.4593 +num rollout transitions: 5000, reward mean: 4.4726 +num rollout transitions: 5000, reward mean: 4.4687 +num rollout transitions: 5000, reward mean: 4.4842 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.4804 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.4611 +num rollout transitions: 5000, reward mean: 4.5361 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.5317 +num rollout transitions: 5000, reward mean: 4.4819 +num rollout transitions: 5000, reward mean: 4.4753 +num rollout transitions: 5000, reward mean: 4.4640 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.4709 +num rollout transitions: 5000, reward mean: 4.4644 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.4728 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.4882 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4644 +num rollout transitions: 5000, reward mean: 4.4786 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.4852 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4858 +num rollout transitions: 5000, reward mean: 4.4747 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.5192 +num rollout transitions: 5000, reward mean: 4.4956 +---------------------------------------------------------------------------------- +| alpha | 0.203 | +| eval/episode_length | 659 | +| eval/episode_length_std | 203 | +| eval/normalized_episode_reward | 65 | +| eval/normalized_episode_reward_std | 22.2 | +| loss/actor | -345 | +| loss/alpha | 0.00862 | +| loss/critic | 70.6 | +| penalty/lcb_mean | 1.87 | +| penalty/lcb_std | 2.68 | +| rollout_info/dynamics_penalty_mean | 0.00218 | +| rollout_info/dynamics_penalty_std | 0.0449 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.4557 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5203 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.4799 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.4944 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5249 +num rollout transitions: 5000, reward mean: 4.5329 +num rollout transitions: 5000, reward mean: 4.5339 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.4746 +num rollout transitions: 5000, reward mean: 4.4633 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.4634 +num rollout transitions: 5000, reward mean: 4.4732 +num rollout transitions: 5000, reward mean: 4.4736 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.4805 +num rollout transitions: 5000, reward mean: 4.4731 +num rollout transitions: 5000, reward mean: 4.5309 +num rollout transitions: 5000, reward mean: 4.5152 +----------------------------------------------------------------------------------- +| alpha | 0.23 | +| eval/episode_length | 131 | +| eval/episode_length_std | 129 | +| eval/normalized_episode_reward | 2.29 | +| eval/normalized_episode_reward_std | 12.6 | +| loss/actor | -337 | +| loss/alpha | -0.000735 | +| loss/critic | 72.7 | +| penalty/lcb_mean | 1.9 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0.00271 | +| rollout_info/dynamics_penalty_std | 0.0497 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1600000 | +----------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.5281 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.4838 +num rollout transitions: 5000, reward mean: 4.4697 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.4872 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.4765 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.4753 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.4805 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.4845 +num rollout transitions: 5000, reward mean: 4.4727 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.4925 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.4818 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.5249 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.5225 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4324 +num rollout transitions: 5000, reward mean: 4.4571 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.4654 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.5096 +num rollout transitions: 5000, reward mean: 4.5259 +num rollout transitions: 5000, reward mean: 4.5226 +---------------------------------------------------------------------------------- +| alpha | 0.225 | +| eval/episode_length | 576 | +| eval/episode_length_std | 443 | +| eval/normalized_episode_reward | 51.4 | +| eval/normalized_episode_reward_std | 46.3 | +| loss/actor | -347 | +| loss/alpha | 0.00694 | +| loss/critic | 80.5 | +| penalty/lcb_mean | 2.01 | +| penalty/lcb_std | 2.7 | +| rollout_info/dynamics_penalty_mean | 0.00327 | +| rollout_info/dynamics_penalty_std | 0.0549 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4808 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.4930 +num rollout transitions: 5000, reward mean: 4.4796 +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.4672 +num rollout transitions: 5000, reward mean: 4.5388 +num rollout transitions: 5000, reward mean: 4.4564 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.4714 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.4801 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.4927 +num rollout transitions: 5000, reward mean: 4.5265 +num rollout transitions: 5000, reward mean: 4.5157 +num rollout transitions: 5000, reward mean: 4.4735 +num rollout transitions: 5000, reward mean: 4.4834 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.4733 +num rollout transitions: 5000, reward mean: 4.4733 +num rollout transitions: 5000, reward mean: 4.4653 +num rollout transitions: 5000, reward mean: 4.4777 +num rollout transitions: 5000, reward mean: 4.5238 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.4879 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5203 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5196 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.5044 +---------------------------------------------------------------------------------- +| alpha | 0.213 | +| eval/episode_length | 343 | +| eval/episode_length_std | 275 | +| eval/normalized_episode_reward | 17.2 | +| eval/normalized_episode_reward_std | 29.5 | +| loss/actor | -345 | +| loss/alpha | -0.00945 | +| loss/critic | 69.9 | +| penalty/lcb_mean | 1.85 | +| penalty/lcb_std | 2.51 | +| rollout_info/dynamics_penalty_mean | 0.00365 | +| rollout_info/dynamics_penalty_std | 0.0582 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.5192 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.4845 +num rollout transitions: 5000, reward mean: 4.4676 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.4815 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.4539 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.4820 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.4785 +num rollout transitions: 5000, reward mean: 4.4510 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.4879 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.4616 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.5205 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.5332 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.4841 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.5333 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5523 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.5022 +---------------------------------------------------------------------------------- +| alpha | 0.222 | +| eval/episode_length | 662 | +| eval/episode_length_std | 206 | +| eval/normalized_episode_reward | 58.3 | +| eval/normalized_episode_reward_std | 22.1 | +| loss/actor | -342 | +| loss/alpha | 0.00268 | +| loss/critic | 75.6 | +| penalty/lcb_mean | 1.92 | +| penalty/lcb_std | 2.66 | +| rollout_info/dynamics_penalty_mean | 0.00203 | +| rollout_info/dynamics_penalty_std | 0.043 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.4810 +num rollout transitions: 5000, reward mean: 4.4820 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.4611 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.4683 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5245 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.4530 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.4655 +num rollout transitions: 5000, reward mean: 4.4795 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.4845 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.5412 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.4841 +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.4904 +num rollout transitions: 5000, reward mean: 4.5178 +num rollout transitions: 5000, reward mean: 4.5300 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.4863 +---------------------------------------------------------------------------------- +| alpha | 0.215 | +| eval/episode_length | 523 | +| eval/episode_length_std | 335 | +| eval/normalized_episode_reward | 51 | +| eval/normalized_episode_reward_std | 36.9 | +| loss/actor | -346 | +| loss/alpha | 0.0085 | +| loss/critic | 73.5 | +| penalty/lcb_mean | 1.89 | +| penalty/lcb_std | 2.67 | +| rollout_info/dynamics_penalty_mean | 0.00183 | +| rollout_info/dynamics_penalty_std | 0.0407 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4578 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.4817 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4940 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.4515 +num rollout transitions: 5000, reward mean: 4.4879 +num rollout transitions: 5000, reward mean: 4.5181 +num rollout transitions: 5000, reward mean: 4.4824 +num rollout transitions: 5000, reward mean: 4.4804 +num rollout transitions: 5000, reward mean: 4.4702 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.4697 +num rollout transitions: 5000, reward mean: 4.4593 +num rollout transitions: 5000, reward mean: 4.4724 +num rollout transitions: 5000, reward mean: 4.4755 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.4664 +num rollout transitions: 5000, reward mean: 4.4568 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.4802 +num rollout transitions: 5000, reward mean: 4.4748 +num rollout transitions: 5000, reward mean: 4.4884 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.5296 +num rollout transitions: 5000, reward mean: 4.4879 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.5331 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.4729 +num rollout transitions: 5000, reward mean: 4.4745 +---------------------------------------------------------------------------------- +| alpha | 0.201 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 0.739 | +| loss/actor | -345 | +| loss/alpha | -0.0139 | +| loss/critic | 72.8 | +| penalty/lcb_mean | 1.89 | +| penalty/lcb_std | 2.73 | +| rollout_info/dynamics_penalty_mean | 0.00291 | +| rollout_info/dynamics_penalty_std | 0.0512 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.5291 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.5170 +num rollout transitions: 5000, reward mean: 4.5430 +num rollout transitions: 5000, reward mean: 4.4624 +num rollout transitions: 5000, reward mean: 4.5216 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.5252 +num rollout transitions: 5000, reward mean: 4.5260 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.4750 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.4819 +num rollout transitions: 5000, reward mean: 4.5017 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4846 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.5350 +num rollout transitions: 5000, reward mean: 4.4643 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.4647 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.4884 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.4930 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.4664 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.5242 +---------------------------------------------------------------------------------- +| alpha | 0.205 | +| eval/episode_length | 945 | +| eval/episode_length_std | 176 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 21.5 | +| loss/actor | -342 | +| loss/alpha | 0.0121 | +| loss/critic | 77.3 | +| penalty/lcb_mean | 1.94 | +| penalty/lcb_std | 2.72 | +| rollout_info/dynamics_penalty_mean | 0.00188 | +| rollout_info/dynamics_penalty_std | 0.0411 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.5361 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.4746 +num rollout transitions: 5000, reward mean: 4.4532 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.4844 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.5296 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5247 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.4840 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.4740 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.4470 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.5157 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.4630 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.4930 +---------------------------------------------------------------------------------- +| alpha | 0.211 | +| eval/episode_length | 33.3 | +| eval/episode_length_std | 3.14 | +| eval/normalized_episode_reward | -0.158 | +| eval/normalized_episode_reward_std | 0.00893 | +| loss/actor | -342 | +| loss/alpha | 0.00671 | +| loss/critic | 83 | +| penalty/lcb_mean | 2.05 | +| penalty/lcb_std | 2.84 | +| rollout_info/dynamics_penalty_mean | 0.00207 | +| rollout_info/dynamics_penalty_std | 0.0438 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.5286 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.4927 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4782 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.4782 +num rollout transitions: 5000, reward mean: 4.4787 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.4567 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.4825 +num rollout transitions: 5000, reward mean: 4.5237 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.4637 +num rollout transitions: 5000, reward mean: 4.4548 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.4760 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.5327 +num rollout transitions: 5000, reward mean: 4.4878 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.4804 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.4800 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.5252 +---------------------------------------------------------------------------------- +| alpha | 0.229 | +| eval/episode_length | 9.44 | +| eval/episode_length_std | 0.535 | +| eval/normalized_episode_reward | -0.119 | +| eval/normalized_episode_reward_std | -0.0267 | +| loss/actor | -339 | +| loss/alpha | -0.0025 | +| loss/critic | 96 | +| penalty/lcb_mean | 2.17 | +| penalty/lcb_std | 3.14 | +| rollout_info/dynamics_penalty_mean | 0.00258 | +| rollout_info/dynamics_penalty_std | 0.0485 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5527 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.5203 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.5383 +num rollout transitions: 5000, reward mean: 4.5230 +num rollout transitions: 5000, reward mean: 4.5345 +num rollout transitions: 5000, reward mean: 4.5360 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.5188 +num rollout transitions: 5000, reward mean: 4.5275 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5342 +num rollout transitions: 5000, reward mean: 4.5270 +num rollout transitions: 5000, reward mean: 4.5450 +num rollout transitions: 5000, reward mean: 4.5297 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.5464 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.4828 +num rollout transitions: 5000, reward mean: 4.5316 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.5411 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.4757 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.4801 +num rollout transitions: 5000, reward mean: 4.4882 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.4755 +num rollout transitions: 5000, reward mean: 4.4642 +num rollout transitions: 5000, reward mean: 4.4405 +---------------------------------------------------------------------------------- +| alpha | 0.203 | +| eval/episode_length | 12 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | -0.356 | +| eval/normalized_episode_reward_std | -0.0317 | +| loss/actor | -325 | +| loss/alpha | 0.00422 | +| loss/critic | 78.2 | +| penalty/lcb_mean | 2 | +| penalty/lcb_std | 2.8 | +| rollout_info/dynamics_penalty_mean | 0.00226 | +| rollout_info/dynamics_penalty_std | 0.0456 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.5379 +num rollout transitions: 5000, reward mean: 4.5213 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.4851 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.4750 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.4868 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.5280 +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.4697 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.5424 +num rollout transitions: 5000, reward mean: 4.4820 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5502 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.5373 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.4902 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5256 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.5029 +---------------------------------------------------------------------------------- +| alpha | 0.23 | +| eval/episode_length | 205 | +| eval/episode_length_std | 304 | +| eval/normalized_episode_reward | 17.6 | +| eval/normalized_episode_reward_std | 33.5 | +| loss/actor | -321 | +| loss/alpha | -0.00572 | +| loss/critic | 92.3 | +| penalty/lcb_mean | 2.2 | +| penalty/lcb_std | 3.05 | +| rollout_info/dynamics_penalty_mean | 0.00208 | +| rollout_info/dynamics_penalty_std | 0.0434 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.4785 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5273 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5249 +num rollout transitions: 5000, reward mean: 4.5247 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.5409 +num rollout transitions: 5000, reward mean: 4.5296 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5106 +num rollout transitions: 5000, reward mean: 4.5096 +num rollout transitions: 5000, reward mean: 4.4742 +num rollout transitions: 5000, reward mean: 4.5227 +num rollout transitions: 5000, reward mean: 4.5471 +num rollout transitions: 5000, reward mean: 4.5441 +num rollout transitions: 5000, reward mean: 4.5562 +num rollout transitions: 5000, reward mean: 4.5362 +num rollout transitions: 5000, reward mean: 4.5376 +num rollout transitions: 5000, reward mean: 4.5332 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.4678 +num rollout transitions: 5000, reward mean: 4.5241 +num rollout transitions: 5000, reward mean: 4.5230 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5213 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5222 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.4768 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.4665 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.4728 +num rollout transitions: 5000, reward mean: 4.5096 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5351 +---------------------------------------------------------------------------------- +| alpha | 0.23 | +| eval/episode_length | 862 | +| eval/episode_length_std | 200 | +| eval/normalized_episode_reward | 92.7 | +| eval/normalized_episode_reward_std | 24.1 | +| loss/actor | -311 | +| loss/alpha | -0.00443 | +| loss/critic | 89.5 | +| penalty/lcb_mean | 2.12 | +| penalty/lcb_std | 2.94 | +| rollout_info/dynamics_penalty_mean | 0.00233 | +| rollout_info/dynamics_penalty_std | 0.0467 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.4925 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.4717 +num rollout transitions: 5000, reward mean: 4.4788 +num rollout transitions: 5000, reward mean: 4.4925 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5307 +num rollout transitions: 5000, reward mean: 4.5337 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5187 +num rollout transitions: 5000, reward mean: 4.5187 +num rollout transitions: 5000, reward mean: 4.5162 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5524 +num rollout transitions: 5000, reward mean: 4.5320 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.4810 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.4799 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5360 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.4762 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5323 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.5250 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.4740 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.4944 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.5157 +---------------------------------------------------------------------------------- +| alpha | 0.205 | +| eval/episode_length | 882 | +| eval/episode_length_std | 155 | +| eval/normalized_episode_reward | 87 | +| eval/normalized_episode_reward_std | 18.3 | +| loss/actor | -320 | +| loss/alpha | 0.00953 | +| loss/critic | 79.4 | +| penalty/lcb_mean | 2.03 | +| penalty/lcb_std | 2.82 | +| rollout_info/dynamics_penalty_mean | 0.00233 | +| rollout_info/dynamics_penalty_std | 0.0461 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.5387 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.5336 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.5430 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.4566 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.4840 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.4700 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.4580 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5205 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.4963 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.4546 +num rollout transitions: 5000, reward mean: 4.4785 +num rollout transitions: 5000, reward mean: 4.4679 +---------------------------------------------------------------------------------- +| alpha | 0.221 | +| eval/episode_length | 620 | +| eval/episode_length_std | 451 | +| eval/normalized_episode_reward | 61.7 | +| eval/normalized_episode_reward_std | 50 | +| loss/actor | -332 | +| loss/alpha | 0.00114 | +| loss/critic | 94.6 | +| penalty/lcb_mean | 2.22 | +| penalty/lcb_std | 3.1 | +| rollout_info/dynamics_penalty_mean | 0.00312 | +| rollout_info/dynamics_penalty_std | 0.0536 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4732 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5230 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.5435 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.5487 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.4802 +num rollout transitions: 5000, reward mean: 4.4927 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.4731 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.4773 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.4802 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.4702 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.5266 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.4469 +num rollout transitions: 5000, reward mean: 4.5257 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.4411 +num rollout transitions: 5000, reward mean: 4.5394 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.4802 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.4694 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.4628 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.4994 +---------------------------------------------------------------------------------- +| alpha | 0.253 | +| eval/episode_length | 618 | +| eval/episode_length_std | 357 | +| eval/normalized_episode_reward | 62.2 | +| eval/normalized_episode_reward_std | 42.9 | +| loss/actor | -340 | +| loss/alpha | 0.00524 | +| loss/critic | 123 | +| penalty/lcb_mean | 2.68 | +| penalty/lcb_std | 3.43 | +| rollout_info/dynamics_penalty_mean | 0.00282 | +| rollout_info/dynamics_penalty_std | 0.0501 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.5230 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.4707 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4731 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.5419 +num rollout transitions: 5000, reward mean: 4.4799 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.4722 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5412 +num rollout transitions: 5000, reward mean: 4.5019 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.4786 +num rollout transitions: 5000, reward mean: 4.4675 +num rollout transitions: 5000, reward mean: 4.4729 +num rollout transitions: 5000, reward mean: 4.4805 +num rollout transitions: 5000, reward mean: 4.4524 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.4819 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.5326 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5278 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.5187 +num rollout transitions: 5000, reward mean: 4.4718 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.4694 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4745 +num rollout transitions: 5000, reward mean: 4.4878 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.4852 +---------------------------------------------------------------------------------- +| alpha | 0.228 | +| eval/episode_length | 63.1 | +| eval/episode_length_std | 1.03 | +| eval/normalized_episode_reward | 0.229 | +| eval/normalized_episode_reward_std | 0.0507 | +| loss/actor | -326 | +| loss/alpha | -0.0115 | +| loss/critic | 101 | +| penalty/lcb_mean | 2.34 | +| penalty/lcb_std | 3.2 | +| rollout_info/dynamics_penalty_mean | 0.00232 | +| rollout_info/dynamics_penalty_std | 0.0455 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.4702 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.5337 +num rollout transitions: 5000, reward mean: 4.4770 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.5295 +num rollout transitions: 5000, reward mean: 4.5283 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.4638 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.4826 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.5021 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.4622 +num rollout transitions: 5000, reward mean: 4.4601 +num rollout transitions: 5000, reward mean: 4.4848 +num rollout transitions: 5000, reward mean: 4.4670 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.4618 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.4796 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.4797 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.4815 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.4829 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.5292 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.4963 +num rollout transitions: 5000, reward mean: 4.4805 +num rollout transitions: 5000, reward mean: 4.4923 +num rollout transitions: 5000, reward mean: 4.4622 +num rollout transitions: 5000, reward mean: 4.5262 +num rollout transitions: 5000, reward mean: 4.4696 +num rollout transitions: 5000, reward mean: 4.4780 +num rollout transitions: 5000, reward mean: 4.5170 +---------------------------------------------------------------------------------- +| alpha | 0.252 | +| eval/episode_length | 37.4 | +| eval/episode_length_std | 3.03 | +| eval/normalized_episode_reward | 0.113 | +| eval/normalized_episode_reward_std | -0.00209 | +| loss/actor | -357 | +| loss/alpha | 0.00817 | +| loss/critic | 126 | +| penalty/lcb_mean | 2.67 | +| penalty/lcb_std | 3.39 | +| rollout_info/dynamics_penalty_mean | 0.00287 | +| rollout_info/dynamics_penalty_std | 0.0513 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5290 +num rollout transitions: 5000, reward mean: 4.4645 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.4858 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.4697 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.4955 +num rollout transitions: 5000, reward mean: 4.4767 +num rollout transitions: 5000, reward mean: 4.4363 +num rollout transitions: 5000, reward mean: 4.4708 +num rollout transitions: 5000, reward mean: 4.4848 +num rollout transitions: 5000, reward mean: 4.4695 +num rollout transitions: 5000, reward mean: 4.4860 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.4844 +num rollout transitions: 5000, reward mean: 4.4658 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.4715 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.4745 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.5332 +num rollout transitions: 5000, reward mean: 4.4579 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.4605 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.4858 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.4715 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.4407 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4643 +num rollout transitions: 5000, reward mean: 4.4688 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.4747 +---------------------------------------------------------------------------------- +| alpha | 0.258 | +| eval/episode_length | 267 | +| eval/episode_length_std | 263 | +| eval/normalized_episode_reward | 14.6 | +| eval/normalized_episode_reward_std | 20.7 | +| loss/actor | -372 | +| loss/alpha | -0.00798 | +| loss/critic | 129 | +| penalty/lcb_mean | 2.76 | +| penalty/lcb_std | 3.51 | +| rollout_info/dynamics_penalty_mean | 0.00181 | +| rollout_info/dynamics_penalty_std | 0.0409 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4704 +num rollout transitions: 5000, reward mean: 4.4796 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.4689 +num rollout transitions: 5000, reward mean: 4.4709 +num rollout transitions: 5000, reward mean: 4.4629 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.4844 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.5017 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.5247 +num rollout transitions: 5000, reward mean: 4.4822 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4762 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.4780 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.4624 +num rollout transitions: 5000, reward mean: 4.4619 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.4660 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.4534 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.4848 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.4817 +---------------------------------------------------------------------------------- +| alpha | 0.245 | +| eval/episode_length | 12 | +| eval/episode_length_std | 0.196 | +| eval/normalized_episode_reward | -0.163 | +| eval/normalized_episode_reward_std | -0.0286 | +| loss/actor | -337 | +| loss/alpha | 0.0136 | +| loss/critic | 118 | +| penalty/lcb_mean | 2.65 | +| penalty/lcb_std | 3.33 | +| rollout_info/dynamics_penalty_mean | 0.00215 | +| rollout_info/dynamics_penalty_std | 0.0441 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.4767 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.4844 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.4787 +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5375 +num rollout transitions: 5000, reward mean: 4.5392 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.4827 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.4665 +num rollout transitions: 5000, reward mean: 4.5263 +num rollout transitions: 5000, reward mean: 4.4777 +num rollout transitions: 5000, reward mean: 4.4988 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.5264 +num rollout transitions: 5000, reward mean: 4.4584 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5351 +num rollout transitions: 5000, reward mean: 4.5256 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.4930 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.4935 +---------------------------------------------------------------------------------- +| alpha | 0.251 | +| eval/episode_length | 20.5 | +| eval/episode_length_std | 1.81 | +| eval/normalized_episode_reward | -0.0692 | +| eval/normalized_episode_reward_std | 0.00572 | +| loss/actor | -332 | +| loss/alpha | -0.0254 | +| loss/critic | 124 | +| penalty/lcb_mean | 2.71 | +| penalty/lcb_std | 3.35 | +| rollout_info/dynamics_penalty_mean | 0.00248 | +| rollout_info/dynamics_penalty_std | 0.0476 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.4944 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.5197 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.4391 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.4851 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.5309 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5324 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.4787 +num rollout transitions: 5000, reward mean: 4.5286 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5334 +num rollout transitions: 5000, reward mean: 4.5330 +num rollout transitions: 5000, reward mean: 4.5178 +num rollout transitions: 5000, reward mean: 4.4671 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.4802 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.4858 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.4741 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.4779 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.5220 +---------------------------------------------------------------------------------- +| alpha | 0.224 | +| eval/episode_length | 524 | +| eval/episode_length_std | 263 | +| eval/normalized_episode_reward | 50.6 | +| eval/normalized_episode_reward_std | 30.8 | +| loss/actor | -318 | +| loss/alpha | 0.0073 | +| loss/critic | 98.8 | +| penalty/lcb_mean | 2.33 | +| penalty/lcb_std | 3.12 | +| rollout_info/dynamics_penalty_mean | 0.00276 | +| rollout_info/dynamics_penalty_std | 0.0501 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5366 +num rollout transitions: 5000, reward mean: 4.4768 +num rollout transitions: 5000, reward mean: 4.4832 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.4756 +num rollout transitions: 5000, reward mean: 4.5197 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5178 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.4760 +num rollout transitions: 5000, reward mean: 4.4558 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.4701 +num rollout transitions: 5000, reward mean: 4.4639 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.4745 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.5258 +num rollout transitions: 5000, reward mean: 4.4679 +num rollout transitions: 5000, reward mean: 4.4863 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.5256 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.4764 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.5296 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5542 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.5072 +----------------------------------------------------------------------------------- +| alpha | 0.215 | +| eval/episode_length | 766 | +| eval/episode_length_std | 320 | +| eval/normalized_episode_reward | 75.2 | +| eval/normalized_episode_reward_std | 38.6 | +| loss/actor | -317 | +| loss/alpha | -0.000168 | +| loss/critic | 93.1 | +| penalty/lcb_mean | 2.24 | +| penalty/lcb_std | 3.03 | +| rollout_info/dynamics_penalty_mean | 0.00228 | +| rollout_info/dynamics_penalty_std | 0.0456 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2650000 | +----------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.5072 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.4821 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.4532 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.5187 +num rollout transitions: 5000, reward mean: 4.5262 +num rollout transitions: 5000, reward mean: 4.5011 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.5180 +num rollout transitions: 5000, reward mean: 4.5415 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.4878 +num rollout transitions: 5000, reward mean: 4.4842 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.4944 +num rollout transitions: 5000, reward mean: 4.5376 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5284 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.5344 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.5300 +num rollout transitions: 5000, reward mean: 4.5162 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.5372 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.5108 +---------------------------------------------------------------------------------- +| alpha | 0.204 | +| eval/episode_length | 922 | +| eval/episode_length_std | 143 | +| eval/normalized_episode_reward | 96.5 | +| eval/normalized_episode_reward_std | 16.2 | +| loss/actor | -317 | +| loss/alpha | 0.00792 | +| loss/critic | 92.9 | +| penalty/lcb_mean | 2.23 | +| penalty/lcb_std | 3.02 | +| rollout_info/dynamics_penalty_mean | 0.00176 | +| rollout_info/dynamics_penalty_std | 0.0398 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4805 +num rollout transitions: 5000, reward mean: 4.4530 +num rollout transitions: 5000, reward mean: 4.4868 +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.4647 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.4515 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.4902 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.4728 +num rollout transitions: 5000, reward mean: 4.4786 +num rollout transitions: 5000, reward mean: 4.4729 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.4701 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.4682 +num rollout transitions: 5000, reward mean: 4.4587 +num rollout transitions: 5000, reward mean: 4.4708 +num rollout transitions: 5000, reward mean: 4.4756 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.5266 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.4797 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.5279 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.4772 +num rollout transitions: 5000, reward mean: 4.5480 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4775 +num rollout transitions: 5000, reward mean: 4.5306 +num rollout transitions: 5000, reward mean: 4.4791 +num rollout transitions: 5000, reward mean: 4.5262 +num rollout transitions: 5000, reward mean: 4.4762 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5283 +---------------------------------------------------------------------------------- +| alpha | 0.201 | +| eval/episode_length | 610 | +| eval/episode_length_std | 406 | +| eval/normalized_episode_reward | 59.6 | +| eval/normalized_episode_reward_std | 46.1 | +| loss/actor | -331 | +| loss/alpha | -0.0664 | +| loss/critic | 99.5 | +| penalty/lcb_mean | 2.36 | +| penalty/lcb_std | 3.11 | +| rollout_info/dynamics_penalty_mean | 0.00261 | +| rollout_info/dynamics_penalty_std | 0.0488 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5249 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.4644 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.4801 +num rollout transitions: 5000, reward mean: 4.4516 +num rollout transitions: 5000, reward mean: 4.4880 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.4846 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.4780 +num rollout transitions: 5000, reward mean: 4.4848 +num rollout transitions: 5000, reward mean: 4.4858 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.4769 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.4273 +num rollout transitions: 5000, reward mean: 4.4744 +num rollout transitions: 5000, reward mean: 4.4818 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.4963 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.5071 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.5017 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.4894 +---------------------------------------------------------------------------------- +| alpha | 0.199 | +| eval/episode_length | 284 | +| eval/episode_length_std | 264 | +| eval/normalized_episode_reward | 23.8 | +| eval/normalized_episode_reward_std | 32.1 | +| loss/actor | -325 | +| loss/alpha | -0.00132 | +| loss/critic | 105 | +| penalty/lcb_mean | 2.43 | +| penalty/lcb_std | 3.25 | +| rollout_info/dynamics_penalty_mean | 0.00199 | +| rollout_info/dynamics_penalty_std | 0.0426 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.4838 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5257 +num rollout transitions: 5000, reward mean: 4.4683 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.5174 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.4526 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.5300 +num rollout transitions: 5000, reward mean: 4.5398 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.4682 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.4768 +num rollout transitions: 5000, reward mean: 4.4772 +num rollout transitions: 5000, reward mean: 4.4786 +num rollout transitions: 5000, reward mean: 4.4623 +num rollout transitions: 5000, reward mean: 4.4530 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.5154 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.4749 +num rollout transitions: 5000, reward mean: 4.4758 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.5259 +num rollout transitions: 5000, reward mean: 4.5346 +num rollout transitions: 5000, reward mean: 4.5180 +num rollout transitions: 5000, reward mean: 4.5304 +num rollout transitions: 5000, reward mean: 4.5390 +---------------------------------------------------------------------------------- +| alpha | 0.188 | +| eval/episode_length | 894 | +| eval/episode_length_std | 159 | +| eval/normalized_episode_reward | 90.7 | +| eval/normalized_episode_reward_std | 20.1 | +| loss/actor | -317 | +| loss/alpha | 0.0489 | +| loss/critic | 153 | +| penalty/lcb_mean | 2.6 | +| penalty/lcb_std | 4.04 | +| rollout_info/dynamics_penalty_mean | 0.00266 | +| rollout_info/dynamics_penalty_std | 0.0496 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.5326 +num rollout transitions: 5000, reward mean: 4.5367 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.4760 +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5309 +num rollout transitions: 5000, reward mean: 4.4790 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5171 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4612 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5255 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.5170 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5293 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.4904 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.5171 +num rollout transitions: 5000, reward mean: 4.5152 +---------------------------------------------------------------------------------- +| alpha | 0.19 | +| eval/episode_length | 788 | +| eval/episode_length_std | 336 | +| eval/normalized_episode_reward | 85.1 | +| eval/normalized_episode_reward_std | 43.5 | +| loss/actor | -287 | +| loss/alpha | -0.0408 | +| loss/critic | 132 | +| penalty/lcb_mean | 2.69 | +| penalty/lcb_std | 4.19 | +| rollout_info/dynamics_penalty_mean | 0.00304 | +| rollout_info/dynamics_penalty_std | 0.053 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5365 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.4896 +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.5309 +num rollout transitions: 5000, reward mean: 4.5301 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.5708 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.4542 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4550 +num rollout transitions: 5000, reward mean: 4.4833 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.4805 +num rollout transitions: 5000, reward mean: 4.4517 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.5350 +num rollout transitions: 5000, reward mean: 4.4720 +num rollout transitions: 5000, reward mean: 4.4816 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.4703 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5293 +num rollout transitions: 5000, reward mean: 4.5252 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.4767 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.4533 +---------------------------------------------------------------------------------- +| alpha | 0.179 | +| eval/episode_length | 837 | +| eval/episode_length_std | 308 | +| eval/normalized_episode_reward | 89.9 | +| eval/normalized_episode_reward_std | 39.5 | +| loss/actor | -286 | +| loss/alpha | -0.00706 | +| loss/critic | 115 | +| penalty/lcb_mean | 2.59 | +| penalty/lcb_std | 3.7 | +| rollout_info/dynamics_penalty_mean | 0.00267 | +| rollout_info/dynamics_penalty_std | 0.0498 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.4716 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.5384 +num rollout transitions: 5000, reward mean: 4.5262 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5187 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5268 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.4834 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.5237 +num rollout transitions: 5000, reward mean: 4.5418 +num rollout transitions: 5000, reward mean: 4.5013 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.4842 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.4754 +num rollout transitions: 5000, reward mean: 4.4669 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.5353 +num rollout transitions: 5000, reward mean: 4.4723 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.4978 +---------------------------------------------------------------------------------- +| alpha | 0.159 | +| eval/episode_length | 426 | +| eval/episode_length_std | 278 | +| eval/normalized_episode_reward | 34.6 | +| eval/normalized_episode_reward_std | 35.3 | +| loss/actor | -289 | +| loss/alpha | 0.0383 | +| loss/critic | 92.4 | +| penalty/lcb_mean | 2.37 | +| penalty/lcb_std | 3.34 | +| rollout_info/dynamics_penalty_mean | 0.0029 | +| rollout_info/dynamics_penalty_std | 0.052 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 48302.14s diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193338/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193338/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193338/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193338/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..55d0008b8f99fac654be2391cdb8627d7c95065b --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193338/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_neuralode.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 123, "device": "cuda:5", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_123×tamp_26-0112-030558/model", "reward_penalty_coef": 0.3, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/neuralODE", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 5000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "neuralODE", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "neuralODE"} diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193338/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193338/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..2111590be89f9bf8a8e1673e24322510f56cac58 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_123×tamp_26-0506-193338/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.2722834360393905,152.6,38.77421823841197,7.040726539423149,3.923521219731383,-315.4453522211516,-1.6623728320011408,27.608343577929904,1.1659204917145942,1.6067424965139563,0.015460900703090715,0.09780891498999857,5000.0,4.502545684085585,50000 +0.20471871116310397,166.93,44.725441305816084,7.176495808989966,3.9121236421609433,-500.8556371777331,0.24554032940799525,61.08868013761538,1.6616161256265598,2.4964798021626238,0.007153751945190136,0.07277806672819437,5000.0,4.4913955467166,100000 +0.17272896461576237,32.28,5.557121557065311,0.06440886854050756,0.05786634486797998,-436.31453547729416,-0.019732645176537254,41.50247578304266,1.38848804876684,2.1325981341528797,0.004080803747033469,0.0596310751714411,5000.0,4.501045121796544,150000 +0.15540275016218408,127.63,13.547438872347792,7.977738257248679,1.381908807026515,-431.6689221807842,0.023749450259543906,31.50815489994016,1.2676485836422615,1.9669465347098958,0.005676110565751792,0.07149060789010216,5000.0,4.505207834133964,200000 +0.17196245766907967,168.39,14.314255132559289,9.087236758883972,0.9610338083602602,-422.69195262145934,-0.0038586488720214573,38.07100426771255,1.3427053117239183,1.9924616518032536,0.00493662776173465,0.06621767659604876,5000.0,4.5160524011449334,250000 +0.1540338472303736,155.43,53.955213835180004,7.538921702619832,3.363331930642143,-419.407311300656,0.00786344702001669,35.556311820926396,1.311451157116907,1.9576150849700005,0.004095258008070293,0.05905324887746915,5000.0,4.5076471278479495,300000 +0.16845835422575703,192.16,24.60598301226757,11.086601266843662,2.283308413129452,-429.00501211914224,-0.0034947538189963345,40.38459727586786,1.4419041818129978,2.1568753452575242,0.0040952707655634735,0.05936604552272517,5000.0,4.508479384493125,350000 +0.17592551013171895,184.65,10.869567608695391,11.90392562674423,1.178236326891898,-411.4256528564417,0.017993304569907446,40.24151611539866,1.381417174179561,2.082723129030425,0.0026969271333557553,0.04868337295143128,5000.0,4.503216247653899,400000 +0.18530833156376864,267.73,201.90729828314772,15.155904326400476,21.61113250334732,-392.436858890998,-0.017298083063885375,41.29455072803534,1.364929894325745,2.0597736778164224,0.0032429890559501936,0.05344751212432372,5000.0,4.517057368386898,450000 +0.1763645866927511,213.23,11.733588538891244,14.688335732300818,1.0956430311700318,-384.6875142211901,-0.008491944905333039,39.016284973201664,1.3544924277639396,1.974145609202376,0.003392943342337386,0.05564627294569835,5000.0,4.514775973253862,500000 +0.1721888171905275,149.54,2.0852817555428813,9.253883859978584,0.07450746489300263,-389.4484167358396,-0.008206245852262123,39.51961161249154,1.4018020101475748,2.04626843641873,0.0040561366303937686,0.06129897786121736,5000.0,4.502272486779233,550000 +0.17639996790736742,111.47,5.243004863625438,5.250075538313223,0.1476766373083638,-393.45559033264175,0.023397918287944255,42.6839913749503,1.4362737295889945,2.1420229344368287,0.00350675211759517,0.05669299512457786,5000.0,4.50651460700666,600000 +0.19692354221135572,488.39,96.95142030934875,43.64900976138406,11.403939313337913,-386.81335688476844,0.00674639320196585,50.211281925124986,1.517511199874903,2.1947559509766084,0.0033375112633444366,0.05531007050375218,5000.0,4.517305830774366,650000 +0.1943174241009348,574.18,259.86598007434526,54.9332401703562,28.284478139267005,-377.297930580448,0.005309444653429232,48.139822460784586,1.5029664032554844,2.193574611305,0.003123325151307508,0.05335332477509407,5000.0,4.510099068540826,700000 +0.19439912349909466,105.59,1.6618965070063776,4.8240073308095965,0.0564376122458551,-361.9565349176078,-0.004565351327098817,50.82813168205176,1.566654240648746,2.2099119616556426,0.0033656804254706962,0.05577910459849057,5000.0,4.504938719265197,750000 +0.19535838359564778,963.49,131.47748818714174,104.61914112353618,15.356443204094383,-361.32981690796436,-0.0008064695585148358,51.932207659873924,1.5488871971678804,2.233820763331651,0.0032061191198555754,0.05433574985723045,5000.0,4.515633049946708,800000 +0.18644690938710898,101.58,2.7025173449952176,4.573921074791009,0.09675295940313836,-368.4157999279887,-0.007403877398148384,51.65630390476201,1.5201259222829304,2.2700896895934,0.0026832582235112794,0.04909066655194667,5000.0,4.510323857256593,850000 +0.18167861304402455,823.79,307.84753677754185,86.0619587815056,35.5082124460636,-361.69233697814957,-0.0025945754913241463,51.929502276840296,1.510183115175931,2.322425066957497,0.0021198150513479486,0.043736411055477424,5000.0,4.507029068924793,900000 +0.1950903361764528,160.96,13.03374082909431,8.444606116272064,1.1955453553645057,-359.7341428680435,0.007223711433187281,53.830894234161725,1.570738697321423,2.2957087986624307,0.0025948604556024087,0.04885979773454439,5000.0,4.506904754412159,950000 +0.1883281271913659,172.48,7.466565475504786,10.856910680739494,0.671540045618641,-359.1726956402617,0.0016701572437956233,54.447322303619146,1.5765169366765073,2.363872427361007,0.0022135441452637307,0.045165030562289585,5000.0,4.50923342051269,1000000 +0.19703339770436396,685.79,429.500111641429,71.64780271621757,47.851159085987824,-357.7491222802722,0.005448261457197302,56.345981720047554,1.6264482771110624,2.269163908994179,0.002890886395959649,0.051253406071541247,5000.0,4.51333486926811,1050000 +0.18899761199415022,920.76,168.086056530576,90.83235623713547,17.713628783579534,-362.27710790283305,-0.004992633445598306,59.78824077762553,1.7116834812283337,2.448628482754238,0.003326439101108466,0.054784683819233036,5000.0,4.503476085377729,1100000 +0.20442524945557214,1000.0,0.0,95.5885838719722,0.7683723542197709,-353.19195122497814,-0.0009627280127441545,68.92972304176348,1.815329825074677,2.584810623435936,0.0035510043671727167,0.05742649240812193,5000.0,4.50376623428036,1150000 +0.19410868287771813,906.72,165.38585671090502,92.77663921325805,19.230134921520555,-345.58753545288084,0.0024213915227353027,58.16356313255252,1.6604394227242385,2.388175053896883,0.003682613653877284,0.058392603439059526,5000.0,4.5048386249607555,1200000 +0.19104122999430048,166.44,8.592229047226335,12.35845917299724,0.9501932783371747,-340.0326009020985,-0.00431168537519872,59.14860534210234,1.6968990993642963,2.3892906562304685,0.0025018733346695078,0.048335464525929774,5000.0,4.5013093956137045,1250000 +0.20988058311194446,144.08,11.172895775044177,7.585554040645107,0.5993408940512295,-345.3506651574754,0.012941199071891656,68.71847239307513,1.8338944111728592,2.525391947526955,0.0034612609250508244,0.054748883476844194,5000.0,4.504131453861396,1300000 +0.20129601548939943,33.96,9.306900665635151,0.14851300134920395,0.08779389293897409,-333.0790042040985,-0.0023991425810381915,62.0250548946388,1.7156359265733045,2.456863159205884,0.0017365563155710695,0.039810725708675186,5000.0,4.508572826121082,1350000 +0.20513368575304994,521.15,204.24810280636635,48.71410587827887,24.18442215676207,-344.68771453185656,0.002950991339534459,68.85744890724152,1.8393727386236198,2.6317113400435157,0.0026459295335039493,0.04944558751601217,5000.0,4.49707950007499,1400000 +0.20500912163347168,857.6,201.7686794326612,89.87320014035645,22.648547884077203,-355.4006589947519,0.0016692185255326993,67.10701395210272,1.8258371052289246,2.531396170017664,0.0030488750462848695,0.0522095202786299,5000.0,4.50812365265366,1450000 +0.20139144885480592,788.38,229.48419466272617,78.96222306884157,26.749786339203396,-359.74225159362805,-0.0017097484892978948,71.03022487522182,1.9098430121993974,2.613530642094614,0.0022862674601972113,0.045495342163133515,5000.0,4.496778346609952,1500000 +0.20323086836725537,659.31,203.47637184695424,64.95505657438669,22.227326012299812,-345.1889016778602,0.008622408561110484,70.56025314178368,1.8706405232620282,2.679383630907541,0.0021819591627717017,0.04486232743509166,5000.0,4.490672314117298,1550000 +0.2301709062018977,130.72,129.28387989227429,2.287082931001428,12.635561159617973,-336.63306225342274,-0.0007348370909319518,72.7302518061451,1.8980074966907499,2.6137176503419797,0.002706441189063713,0.049692776538189,5000.0,4.498691454906306,1600000 +0.2245269340699917,575.81,443.459303544305,51.423399407778035,46.27040849919176,-346.8785369512897,0.006943818580508451,80.50414377723658,2.0076564649319733,2.69991454074376,0.0032742612703815102,0.05491310510281863,5000.0,4.496922399073106,1650000 +0.21306074462652153,342.93,274.87307088909233,17.15783444029121,29.463222565486213,-344.57676506347116,-0.009453868061751138,69.88952062847122,1.8538009909391264,2.5087749899327787,0.0036525297841038533,0.058225820307386246,5000.0,4.497581512421639,1700000 +0.22150895790696262,662.11,206.37964507189173,58.296955353891164,22.08903987724012,-341.66151279602,0.0026807665213192753,75.63359689323276,1.9243028913736258,2.6615567866706513,0.002031813997952734,0.042997965117413495,5000.0,4.499368063572343,1750000 +0.21492933526515742,523.41,335.44585539249107,50.97951663162081,36.866445530849305,-345.5142019799728,0.00850326249837861,73.46359690269404,1.8860371802878402,2.6681859454917953,0.0018256555418893696,0.0406879242649,5000.0,4.499744347232069,1800000 +0.20121002345740832,1000.0,0.0,105.37293590413877,0.7390692390705279,-344.8048972271774,-0.01393059063997094,72.83734051246695,1.8864208747577726,2.731904936671261,0.00291014800251834,0.05123671217064248,5000.0,4.4911218498119165,1850000 +0.20465699194580025,944.89,176.0824179184282,105.00250063661454,21.493841079871036,-341.52680243408537,0.012052441437877766,77.32891975021411,1.9413789231395635,2.7184130500435693,0.0018827556526437395,0.04108882072443153,5000.0,4.500953339858341,1900000 +0.21111557458072835,33.34,3.137578684272316,-0.1576876649633576,0.008934724481697966,-341.89318084045186,0.0067082946424933555,82.95883186511918,2.0470292933726317,2.838371559839253,0.002073074311273172,0.043809656926037686,5000.0,4.499462637808709,1950000 +0.22865593261689157,9.44,0.5351635264103861,-0.11906660417978737,-0.026681766309304276,-338.62076164062603,-0.0024994636437295843,96.04937831604069,2.1654478430222674,3.143744451837591,0.002583986088558363,0.048526351883839455,5000.0,4.499458882017997,2000000 +0.20297929515719437,12.0,0.0,-0.356057069298108,-0.031663616126279145,-324.8827821813965,0.004222735508233658,78.15680301296283,2.0027828404521846,2.8042212625455347,0.0022563866882417353,0.045562879927943514,5000.0,4.5080443918942095,2050000 +0.22985524620234907,205.19,304.2176423220718,17.57978626005122,33.54994008364072,-320.6141291705235,-0.0057173579838872245,92.33304351066614,2.199169371802772,3.047824116561383,0.002078732713289559,0.043416094359650104,5000.0,4.5036219681623555,2100000 +0.23012232244401923,861.74,200.11884568925535,92.66280424782929,24.146568591511265,-311.49419390442307,-0.004428434317354129,89.49386844238339,2.1233702186560555,2.9398863926625034,0.00233443166282773,0.04667419488246545,5000.0,4.510175304168466,2150000 +0.20476140672743126,882.43,154.90837646815615,87.04179452918926,18.3288169114847,-319.5628174688699,0.00953239056855444,79.36031515167211,2.034521460731012,2.821094935376562,0.0023260661899520787,0.04605989498844211,5000.0,4.5051592041185975,2200000 +0.22133927344411664,619.89,451.0758671221505,61.652588478312275,50.04689975458345,-332.0390093548608,0.0011350235999375918,94.64498227519944,2.2166249497151296,3.100267080736171,0.003123977782048285,0.0536332029938698,5000.0,4.5029265318189395,2250000 +0.25328790590166916,617.88,357.3248460434846,62.23882903375735,42.9489882445818,-340.3424447283914,0.005236210126839534,122.9254490404508,2.68073522004605,3.4306712004399507,0.0028171367451231915,0.05008913977064649,5000.0,4.497037458231041,2300000 +0.2276131922209262,63.15,1.0331989159885913,0.22872366273825057,0.050730873912398035,-326.14777327942033,-0.01148265761790774,101.34200023063649,2.336889823808655,3.204683777203556,0.0023162047194428734,0.04547024869696985,5000.0,4.497336366102432,2350000 +0.2524082572615175,37.38,3.0324247723562743,0.11335922527520058,-0.002092862730205391,-356.737588228151,0.008171605237964367,125.65590467269735,2.6699844297265836,3.393980313413137,0.002868287625413388,0.05128695506575191,5000.0,4.496038572560874,2400000 +0.25779542159706836,266.61,262.5796601033674,14.582552910849506,20.66868548031681,-372.3781415179432,-0.0079822676719725,128.7811410230247,2.76247375576735,3.5050952186227136,0.0018072994395084672,0.04085576724555129,5000.0,4.487367006321137,2450000 +0.2454265158584726,12.04,0.19595917942265423,-0.16319103092855447,-0.028600999404466868,-337.2038578863534,0.013566448244042949,118.29976911102194,2.6504363144087915,3.325625859704047,0.002149312418188901,0.044094234721817484,5000.0,4.4921089143840405,2500000 +0.25103101496607094,20.49,1.8082864817279372,-0.06923349858691889,0.005722094849248258,-332.3330415661649,-0.025394214484002458,124.15369981361496,2.7106020511436713,3.353771673743744,0.0024777193687777043,0.04757673488457711,5000.0,4.502681230335961,2550000 +0.22437640915155097,524.47,263.4612098583015,50.62832984513527,30.798723263230116,-317.9517698321556,0.007296447797045059,98.8381906959534,2.325680815658556,3.1167994292641,0.0027637837260714743,0.050143694089543535,5000.0,4.5009567641116,2600000 +0.2149588185670971,765.52,320.3176386026845,75.22454106356518,38.63560360859192,-316.63771551208094,-0.00016773373819881497,93.08061635757456,2.2401570437812497,3.025294323103407,0.002276909077599646,0.04563804869302302,5000.0,4.499771784515881,2650000 +0.2038416371110096,921.62,143.02459788441985,96.47601802020127,16.21738812989445,-316.62606334412084,0.00791742956236008,92.88009707420362,2.2335753560495366,3.0175257313656614,0.0017647163909971712,0.03976168247410089,5000.0,4.503048427753544,2700000 +0.20077343049079088,610.23,405.9754144033848,59.62985600923238,46.067477254623604,-331.4721100280709,-0.06642252677783303,99.48158472427458,2.362954975650306,3.110007637600892,0.0026102545896153893,0.04881696457719154,5000.0,4.494518199865108,2750000 +0.19945250519037233,284.22,264.44684078279323,23.752310710196102,32.14524997627552,-324.7984392315728,-0.0013165652031074582,104.63501831550556,2.4288781726741417,3.2505553984594653,0.0019877629620507374,0.042558729983733654,5000.0,4.496935548687695,2800000 +0.18773873277366124,893.96,159.3215566080121,90.70811927233437,20.05139496037865,-317.1742347961468,0.048903964203261305,153.0516423427612,2.5963709611845127,4.03824561454294,0.002656115696743131,0.04955422508399262,5000.0,4.500246957678133,2850000 +0.19038218940406837,788.32,336.45867145906647,85.05911725206442,43.453428152544454,-287.2525616738889,-0.04075912296544787,132.20785373230157,2.694801356804366,4.185783380813615,0.003043479333965107,0.052963394018711696,5000.0,4.50413444953326,2900000 +0.17903016346126793,837.18,308.06406411654046,89.92855140887335,39.542651449873475,-285.9271664999401,-0.007062824858315581,115.34326139976557,2.593149988939751,3.701225244383805,0.0026726742676515137,0.0497623955333292,5000.0,4.499818826244297,2950000 +0.15933127988532333,426.16,278.2986784014613,34.608146656541436,35.252363482105984,-288.63203353209883,0.03831109126731752,92.41064841079528,2.370846510725031,3.338852659974089,0.0029031170674581083,0.052033132373388816,5000.0,4.500690350752427,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_42×tamp_26-0506-193338/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_42×tamp_26-0506-193338/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..2cb2c1a5874aad036b5ec06f6a4a124231bfb875 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_42×tamp_26-0506-193338/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 5000, reward mean: 4.3748 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.5630 +num rollout transitions: 5000, reward mean: 4.5603 +num rollout transitions: 5000, reward mean: 4.5187 +num rollout transitions: 5000, reward mean: 4.4683 +num rollout transitions: 5000, reward mean: 4.4839 +num rollout transitions: 5000, reward mean: 4.4734 +num rollout transitions: 5000, reward mean: 4.4662 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.4516 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.5385 +num rollout transitions: 5000, reward mean: 4.5309 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.4736 +num rollout transitions: 5000, reward mean: 4.5403 +num rollout transitions: 5000, reward mean: 4.5263 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5524 +num rollout transitions: 5000, reward mean: 4.5565 +num rollout transitions: 5000, reward mean: 4.5390 +num rollout transitions: 5000, reward mean: 4.5563 +num rollout transitions: 5000, reward mean: 4.5376 +num rollout transitions: 5000, reward mean: 4.5641 +num rollout transitions: 5000, reward mean: 4.5323 +num rollout transitions: 5000, reward mean: 4.5463 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.5518 +num rollout transitions: 5000, reward mean: 4.5590 +num rollout transitions: 5000, reward mean: 4.5559 +num rollout transitions: 5000, reward mean: 4.5701 +num rollout transitions: 5000, reward mean: 4.5375 +num rollout transitions: 5000, reward mean: 4.5336 +num rollout transitions: 5000, reward mean: 4.5299 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.5360 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.5368 +num rollout transitions: 5000, reward mean: 4.5393 +num rollout transitions: 5000, reward mean: 4.5591 +num rollout transitions: 5000, reward mean: 4.5734 +num rollout transitions: 5000, reward mean: 4.5164 +---------------------------------------------------------------------------------- +| alpha | 0.306 | +| eval/episode_length | 114 | +| eval/episode_length_std | 15 | +| eval/normalized_episode_reward | 0.826 | +| eval/normalized_episode_reward_std | 0.3 | +| loss/actor | -304 | +| loss/alpha | -1.56 | +| loss/critic | 27.8 | +| penalty/lcb_mean | 1.22 | +| penalty/lcb_std | 1.5 | +| rollout_info/dynamics_penalty_mean | 0.0175 | +| rollout_info/dynamics_penalty_std | 0.107 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.5357 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.5280 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.5305 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.5187 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5305 +num rollout transitions: 5000, reward mean: 4.5376 +num rollout transitions: 5000, reward mean: 4.4766 +num rollout transitions: 5000, reward mean: 4.5194 +num rollout transitions: 5000, reward mean: 4.4904 +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.4828 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.5071 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.5492 +num rollout transitions: 5000, reward mean: 4.5315 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.5395 +num rollout transitions: 5000, reward mean: 4.5434 +num rollout transitions: 5000, reward mean: 4.5400 +num rollout transitions: 5000, reward mean: 4.4787 +num rollout transitions: 5000, reward mean: 4.5372 +num rollout transitions: 5000, reward mean: 4.5011 +num rollout transitions: 5000, reward mean: 4.5414 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.5410 +num rollout transitions: 5000, reward mean: 4.5471 +num rollout transitions: 5000, reward mean: 4.4930 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.5546 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.5565 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5595 +---------------------------------------------------------------------------------- +| alpha | 0.153 | +| eval/episode_length | 191 | +| eval/episode_length_std | 154 | +| eval/normalized_episode_reward | 4.08 | +| eval/normalized_episode_reward_std | 3.94 | +| loss/actor | -422 | +| loss/alpha | -0.0182 | +| loss/critic | 29.7 | +| penalty/lcb_mean | 1.25 | +| penalty/lcb_std | 1.78 | +| rollout_info/dynamics_penalty_mean | 0.00392 | +| rollout_info/dynamics_penalty_std | 0.0567 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.4827 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.5287 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.5277 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.4562 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.4528 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.4316 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.5238 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.5411 +num rollout transitions: 5000, reward mean: 4.5463 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.5463 +num rollout transitions: 5000, reward mean: 4.5293 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.5264 +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.5180 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.5482 +num rollout transitions: 5000, reward mean: 4.5313 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.5304 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.5237 +num rollout transitions: 5000, reward mean: 4.4949 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5272 +---------------------------------------------------------------------------------- +| alpha | 0.157 | +| eval/episode_length | 96 | +| eval/episode_length_std | 3.37 | +| eval/normalized_episode_reward | 4.21 | +| eval/normalized_episode_reward_std | 0.131 | +| loss/actor | -417 | +| loss/alpha | -0.00357 | +| loss/critic | 35.3 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 1.98 | +| rollout_info/dynamics_penalty_mean | 0.00536 | +| rollout_info/dynamics_penalty_std | 0.0682 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5330 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.4746 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.5334 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.5017 +num rollout transitions: 5000, reward mean: 4.5177 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5241 +num rollout transitions: 5000, reward mean: 4.5115 +num rollout transitions: 5000, reward mean: 4.5457 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.4788 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.4681 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5354 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.4686 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5576 +num rollout transitions: 5000, reward mean: 4.4988 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.4771 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.4592 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5417 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.4720 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.5213 +num rollout transitions: 5000, reward mean: 4.5203 +---------------------------------------------------------------------------------- +| alpha | 0.158 | +| eval/episode_length | 98.8 | +| eval/episode_length_std | 1.6 | +| eval/normalized_episode_reward | 4.37 | +| eval/normalized_episode_reward_std | 0.0126 | +| loss/actor | -409 | +| loss/alpha | 0.0265 | +| loss/critic | 36.9 | +| penalty/lcb_mean | 1.43 | +| penalty/lcb_std | 1.98 | +| rollout_info/dynamics_penalty_mean | 0.00612 | +| rollout_info/dynamics_penalty_std | 0.0735 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.5196 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.5096 +num rollout transitions: 5000, reward mean: 4.5168 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5258 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5450 +num rollout transitions: 5000, reward mean: 4.4825 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.5349 +num rollout transitions: 5000, reward mean: 4.5508 +num rollout transitions: 5000, reward mean: 4.5440 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.5457 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5227 +num rollout transitions: 5000, reward mean: 4.5479 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5448 +num rollout transitions: 5000, reward mean: 4.5376 +num rollout transitions: 5000, reward mean: 4.5266 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.4940 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.5424 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5217 +num rollout transitions: 5000, reward mean: 4.5213 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.5126 +---------------------------------------------------------------------------------- +| alpha | 0.183 | +| eval/episode_length | 185 | +| eval/episode_length_std | 4.29 | +| eval/normalized_episode_reward | 8.42 | +| eval/normalized_episode_reward_std | 0.137 | +| loss/actor | -391 | +| loss/alpha | 0.00336 | +| loss/critic | 41.7 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.08 | +| rollout_info/dynamics_penalty_mean | 0.00521 | +| rollout_info/dynamics_penalty_std | 0.069 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.4790 +num rollout transitions: 5000, reward mean: 4.5247 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5377 +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.5567 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.4829 +num rollout transitions: 5000, reward mean: 4.5347 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.4940 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.5360 +num rollout transitions: 5000, reward mean: 4.5406 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5393 +num rollout transitions: 5000, reward mean: 4.5276 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.5366 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5230 +num rollout transitions: 5000, reward mean: 4.5257 +num rollout transitions: 5000, reward mean: 4.5240 +num rollout transitions: 5000, reward mean: 4.5427 +num rollout transitions: 5000, reward mean: 4.5306 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.5106 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.5511 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5639 +num rollout transitions: 5000, reward mean: 4.5213 +num rollout transitions: 5000, reward mean: 4.4930 +---------------------------------------------------------------------------------- +| alpha | 0.178 | +| eval/episode_length | 165 | +| eval/episode_length_std | 18.4 | +| eval/normalized_episode_reward | 11.2 | +| eval/normalized_episode_reward_std | 1.81 | +| loss/actor | -379 | +| loss/alpha | -0.00745 | +| loss/critic | 43 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.07 | +| rollout_info/dynamics_penalty_mean | 0.00275 | +| rollout_info/dynamics_penalty_std | 0.0492 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.4904 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5175 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.4846 +num rollout transitions: 5000, reward mean: 4.5366 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.4786 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.4743 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.5271 +num rollout transitions: 5000, reward mean: 4.5473 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5398 +num rollout transitions: 5000, reward mean: 4.4844 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.5557 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5217 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.5165 +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.4955 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.5019 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.5604 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5230 +num rollout transitions: 5000, reward mean: 4.5268 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.5188 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5364 +num rollout transitions: 5000, reward mean: 4.5273 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5424 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.4925 +num rollout transitions: 5000, reward mean: 4.5281 +---------------------------------------------------------------------------------- +| alpha | 0.186 | +| eval/episode_length | 118 | +| eval/episode_length_std | 2.65 | +| eval/normalized_episode_reward | 5.65 | +| eval/normalized_episode_reward_std | 0.135 | +| loss/actor | -372 | +| loss/alpha | -0.00034 | +| loss/critic | 41.6 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.08 | +| rollout_info/dynamics_penalty_mean | 0.00261 | +| rollout_info/dynamics_penalty_std | 0.0488 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.4884 +num rollout transitions: 5000, reward mean: 4.5217 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.5265 +num rollout transitions: 5000, reward mean: 4.5017 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.5342 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5365 +num rollout transitions: 5000, reward mean: 4.5310 +num rollout transitions: 5000, reward mean: 4.5370 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5339 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5484 +num rollout transitions: 5000, reward mean: 4.5290 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5096 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5393 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.4833 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5256 +num rollout transitions: 5000, reward mean: 4.5245 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.4923 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.5199 +---------------------------------------------------------------------------------- +| alpha | 0.179 | +| eval/episode_length | 270 | +| eval/episode_length_std | 266 | +| eval/normalized_episode_reward | 23.7 | +| eval/normalized_episode_reward_std | 30.2 | +| loss/actor | -371 | +| loss/alpha | 0.0143 | +| loss/critic | 41.5 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.07 | +| rollout_info/dynamics_penalty_mean | 0.00314 | +| rollout_info/dynamics_penalty_std | 0.0531 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.5341 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5300 +num rollout transitions: 5000, reward mean: 4.5566 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.5437 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.5258 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.5238 +num rollout transitions: 5000, reward mean: 4.5459 +num rollout transitions: 5000, reward mean: 4.5363 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.5371 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.4773 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.5593 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4902 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.5458 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.5337 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.5317 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.5276 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.4881 +---------------------------------------------------------------------------------- +| alpha | 0.195 | +| eval/episode_length | 923 | +| eval/episode_length_std | 189 | +| eval/normalized_episode_reward | 90.4 | +| eval/normalized_episode_reward_std | 19.9 | +| loss/actor | -369 | +| loss/alpha | -0.00112 | +| loss/critic | 48.8 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 0.00343 | +| rollout_info/dynamics_penalty_std | 0.0555 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4543 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5299 +num rollout transitions: 5000, reward mean: 4.5227 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.4515 +num rollout transitions: 5000, reward mean: 4.4842 +num rollout transitions: 5000, reward mean: 4.4743 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.4865 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.4816 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.5275 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.4794 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5196 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.4784 +num rollout transitions: 5000, reward mean: 4.4791 +num rollout transitions: 5000, reward mean: 4.5365 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.5128 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.4495 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5128 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.4716 +num rollout transitions: 5000, reward mean: 4.4846 +num rollout transitions: 5000, reward mean: 4.4612 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5017 +---------------------------------------------------------------------------------- +| alpha | 0.194 | +| eval/episode_length | 125 | +| eval/episode_length_std | 2.08 | +| eval/normalized_episode_reward | 7.08 | +| eval/normalized_episode_reward_std | 0.104 | +| loss/actor | -367 | +| loss/alpha | 0.0019 | +| loss/critic | 51.7 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.37 | +| rollout_info/dynamics_penalty_mean | 0.00661 | +| rollout_info/dynamics_penalty_std | 0.0765 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.5096 +num rollout transitions: 5000, reward mean: 4.5268 +num rollout transitions: 5000, reward mean: 4.5305 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.5387 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5509 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4646 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.5277 +num rollout transitions: 5000, reward mean: 4.4640 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.4730 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.4735 +num rollout transitions: 5000, reward mean: 4.5295 +num rollout transitions: 5000, reward mean: 4.4743 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.5335 +num rollout transitions: 5000, reward mean: 4.4800 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.5013 +num rollout transitions: 5000, reward mean: 4.5336 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5465 +num rollout transitions: 5000, reward mean: 4.5449 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.5390 +num rollout transitions: 5000, reward mean: 4.5262 +num rollout transitions: 5000, reward mean: 4.4808 +num rollout transitions: 5000, reward mean: 4.4625 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.4799 +num rollout transitions: 5000, reward mean: 4.5275 +---------------------------------------------------------------------------------- +| alpha | 0.187 | +| eval/episode_length | 122 | +| eval/episode_length_std | 3.04 | +| eval/normalized_episode_reward | 6.68 | +| eval/normalized_episode_reward_std | 0.259 | +| loss/actor | -358 | +| loss/alpha | -0.00837 | +| loss/critic | 48.5 | +| penalty/lcb_mean | 1.56 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 0.00533 | +| rollout_info/dynamics_penalty_std | 0.0697 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4890 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.5377 +num rollout transitions: 5000, reward mean: 4.5428 +num rollout transitions: 5000, reward mean: 4.5459 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.4834 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4772 +num rollout transitions: 5000, reward mean: 4.4598 +num rollout transitions: 5000, reward mean: 4.4848 +num rollout transitions: 5000, reward mean: 4.4818 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5350 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.5080 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.4817 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.4858 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.4641 +num rollout transitions: 5000, reward mean: 4.4759 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5276 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.5387 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.5532 +num rollout transitions: 5000, reward mean: 4.5017 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.5529 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5267 +---------------------------------------------------------------------------------- +| alpha | 0.205 | +| eval/episode_length | 984 | +| eval/episode_length_std | 110 | +| eval/normalized_episode_reward | 97.6 | +| eval/normalized_episode_reward_std | 11.9 | +| loss/actor | -347 | +| loss/alpha | 0.0126 | +| loss/critic | 53.8 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.36 | +| rollout_info/dynamics_penalty_mean | 0.00459 | +| rollout_info/dynamics_penalty_std | 0.0652 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.4733 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.4860 +num rollout transitions: 5000, reward mean: 4.4707 +num rollout transitions: 5000, reward mean: 4.5196 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.4785 +num rollout transitions: 5000, reward mean: 4.5205 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.4637 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.4708 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4738 +num rollout transitions: 5000, reward mean: 4.4735 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.5165 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.5217 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5599 +num rollout transitions: 5000, reward mean: 4.5247 +num rollout transitions: 5000, reward mean: 4.4802 +num rollout transitions: 5000, reward mean: 4.4721 +num rollout transitions: 5000, reward mean: 4.5276 +num rollout transitions: 5000, reward mean: 4.4988 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.4770 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.4817 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.5381 +num rollout transitions: 5000, reward mean: 4.4643 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5139 +----------------------------------------------------------------------------------- +| alpha | 0.212 | +| eval/episode_length | 660 | +| eval/episode_length_std | 252 | +| eval/normalized_episode_reward | 67.2 | +| eval/normalized_episode_reward_std | 28.2 | +| loss/actor | -337 | +| loss/alpha | -9.09e-05 | +| loss/critic | 55.2 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.00517 | +| rollout_info/dynamics_penalty_std | 0.0696 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 650000 | +----------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4732 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.4878 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.4851 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.4338 +num rollout transitions: 5000, reward mean: 4.4824 +num rollout transitions: 5000, reward mean: 4.4930 +num rollout transitions: 5000, reward mean: 4.4788 +num rollout transitions: 5000, reward mean: 4.4904 +num rollout transitions: 5000, reward mean: 4.5295 +num rollout transitions: 5000, reward mean: 4.5196 +num rollout transitions: 5000, reward mean: 4.5216 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.4797 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5257 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.5268 +num rollout transitions: 5000, reward mean: 4.5495 +num rollout transitions: 5000, reward mean: 4.5321 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5238 +num rollout transitions: 5000, reward mean: 4.4756 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.5162 +num rollout transitions: 5000, reward mean: 4.5014 +---------------------------------------------------------------------------------- +| alpha | 0.192 | +| eval/episode_length | 124 | +| eval/episode_length_std | 12.2 | +| eval/normalized_episode_reward | 6.82 | +| eval/normalized_episode_reward_std | 1.18 | +| loss/actor | -342 | +| loss/alpha | -0.00841 | +| loss/critic | 51.5 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.23 | +| rollout_info/dynamics_penalty_mean | 0.00777 | +| rollout_info/dynamics_penalty_std | 0.0846 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.4872 +num rollout transitions: 5000, reward mean: 4.5240 +num rollout transitions: 5000, reward mean: 4.5302 +num rollout transitions: 5000, reward mean: 4.5324 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5302 +num rollout transitions: 5000, reward mean: 4.5252 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5327 +num rollout transitions: 5000, reward mean: 4.5322 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.5311 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5177 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.4766 +num rollout transitions: 5000, reward mean: 4.5327 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5255 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4865 +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.4726 +num rollout transitions: 5000, reward mean: 4.4778 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4735 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4764 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.5021 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5220 +num rollout transitions: 5000, reward mean: 4.5207 +---------------------------------------------------------------------------------- +| alpha | 0.188 | +| eval/episode_length | 165 | +| eval/episode_length_std | 8.64 | +| eval/normalized_episode_reward | 10.5 | +| eval/normalized_episode_reward_std | 0.701 | +| loss/actor | -340 | +| loss/alpha | 0.00713 | +| loss/critic | 54.4 | +| penalty/lcb_mean | 1.69 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 0.00634 | +| rollout_info/dynamics_penalty_std | 0.0759 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4773 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.5442 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.4872 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.5304 +num rollout transitions: 5000, reward mean: 4.5422 +num rollout transitions: 5000, reward mean: 4.4790 +num rollout transitions: 5000, reward mean: 4.4882 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5335 +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.5368 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5362 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.5594 +num rollout transitions: 5000, reward mean: 4.4700 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.5461 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5330 +num rollout transitions: 5000, reward mean: 4.4787 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.4982 +----------------------------------------------------------------------------------- +| alpha | 0.214 | +| eval/episode_length | 978 | +| eval/episode_length_std | 119 | +| eval/normalized_episode_reward | 94.1 | +| eval/normalized_episode_reward_std | 12.7 | +| loss/actor | -337 | +| loss/alpha | -1.08e-05 | +| loss/critic | 58 | +| penalty/lcb_mean | 1.76 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 0.00387 | +| rollout_info/dynamics_penalty_std | 0.0588 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 800000 | +----------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.5452 +num rollout transitions: 5000, reward mean: 4.4696 +num rollout transitions: 5000, reward mean: 4.4930 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5536 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.4618 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.4744 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.4707 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.4758 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.5106 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5325 +num rollout transitions: 5000, reward mean: 4.4852 +num rollout transitions: 5000, reward mean: 4.5174 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5177 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4564 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.4603 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.4659 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.5096 +---------------------------------------------------------------------------------- +| alpha | 0.201 | +| eval/episode_length | 496 | +| eval/episode_length_std | 417 | +| eval/normalized_episode_reward | 45.6 | +| eval/normalized_episode_reward_std | 42.9 | +| loss/actor | -331 | +| loss/alpha | -0.0169 | +| loss/critic | 58.6 | +| penalty/lcb_mean | 1.77 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 0.00598 | +| rollout_info/dynamics_penalty_std | 0.0732 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.4439 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.4747 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.4760 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5368 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5292 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4879 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.4698 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.4822 +num rollout transitions: 5000, reward mean: 4.5062 +---------------------------------------------------------------------------------- +| alpha | 0.204 | +| eval/episode_length | 526 | +| eval/episode_length_std | 235 | +| eval/normalized_episode_reward | 52.2 | +| eval/normalized_episode_reward_std | 27.4 | +| loss/actor | -328 | +| loss/alpha | 0.0206 | +| loss/critic | 60.2 | +| penalty/lcb_mean | 1.79 | +| penalty/lcb_std | 2.37 | +| rollout_info/dynamics_penalty_mean | 0.00503 | +| rollout_info/dynamics_penalty_std | 0.0677 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5368 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.4775 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.5013 +num rollout transitions: 5000, reward mean: 4.5369 +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.4692 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.4737 +num rollout transitions: 5000, reward mean: 4.4727 +num rollout transitions: 5000, reward mean: 4.4779 +num rollout transitions: 5000, reward mean: 4.4787 +num rollout transitions: 5000, reward mean: 4.5278 +num rollout transitions: 5000, reward mean: 4.5325 +num rollout transitions: 5000, reward mean: 4.5271 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5285 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.4718 +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.4939 +---------------------------------------------------------------------------------- +| alpha | 0.224 | +| eval/episode_length | 906 | +| eval/episode_length_std | 171 | +| eval/normalized_episode_reward | 91.5 | +| eval/normalized_episode_reward_std | 18.8 | +| loss/actor | -321 | +| loss/alpha | -0.00246 | +| loss/critic | 71.1 | +| penalty/lcb_mean | 1.92 | +| penalty/lcb_std | 2.53 | +| rollout_info/dynamics_penalty_mean | 0.00312 | +| rollout_info/dynamics_penalty_std | 0.0527 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.4782 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.4885 +num rollout transitions: 5000, reward mean: 4.4826 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.4598 +num rollout transitions: 5000, reward mean: 4.4817 +num rollout transitions: 5000, reward mean: 4.4603 +num rollout transitions: 5000, reward mean: 4.4839 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5258 +num rollout transitions: 5000, reward mean: 4.4502 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.4825 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5266 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.5276 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5302 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.4695 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.4801 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.4816 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.4860 +num rollout transitions: 5000, reward mean: 4.4916 +---------------------------------------------------------------------------------- +| alpha | 0.212 | +| eval/episode_length | 245 | +| eval/episode_length_std | 320 | +| eval/normalized_episode_reward | 20.7 | +| eval/normalized_episode_reward_std | 34.3 | +| loss/actor | -318 | +| loss/alpha | 0.00428 | +| loss/critic | 60.1 | +| penalty/lcb_mean | 1.78 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 0.00407 | +| rollout_info/dynamics_penalty_std | 0.0609 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4719 +num rollout transitions: 5000, reward mean: 4.4775 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.4832 +num rollout transitions: 5000, reward mean: 4.4665 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.4792 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.4697 +num rollout transitions: 5000, reward mean: 4.5358 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.4820 +num rollout transitions: 5000, reward mean: 4.4717 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.5394 +num rollout transitions: 5000, reward mean: 4.4848 +num rollout transitions: 5000, reward mean: 4.4820 +num rollout transitions: 5000, reward mean: 4.5096 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.5192 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.4675 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.4615 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.5106 +num rollout transitions: 5000, reward mean: 4.5294 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.5248 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.4705 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.5125 +---------------------------------------------------------------------------------- +| alpha | 0.214 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 1.11 | +| loss/actor | -326 | +| loss/alpha | -0.00712 | +| loss/critic | 57.9 | +| penalty/lcb_mean | 1.76 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 0.00408 | +| rollout_info/dynamics_penalty_std | 0.0619 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4816 +num rollout transitions: 5000, reward mean: 4.4532 +num rollout transitions: 5000, reward mean: 4.4775 +num rollout transitions: 5000, reward mean: 4.5281 +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.5592 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.5338 +num rollout transitions: 5000, reward mean: 4.5292 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.4555 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.4860 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5220 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.5376 +num rollout transitions: 5000, reward mean: 4.4580 +num rollout transitions: 5000, reward mean: 4.4538 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.4689 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.4649 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5416 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.4706 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.5256 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.5273 +num rollout transitions: 5000, reward mean: 4.4942 +---------------------------------------------------------------------------------- +| alpha | 0.214 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 0.815 | +| loss/actor | -323 | +| loss/alpha | 0.00831 | +| loss/critic | 59.2 | +| penalty/lcb_mean | 1.79 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 0.00504 | +| rollout_info/dynamics_penalty_std | 0.0688 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5290 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4568 +num rollout transitions: 5000, reward mean: 4.4944 +num rollout transitions: 5000, reward mean: 4.4806 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.4679 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.4730 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.4772 +num rollout transitions: 5000, reward mean: 4.4341 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.4762 +num rollout transitions: 5000, reward mean: 4.4902 +num rollout transitions: 5000, reward mean: 4.4825 +num rollout transitions: 5000, reward mean: 4.4748 +num rollout transitions: 5000, reward mean: 4.4863 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.4842 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.5607 +num rollout transitions: 5000, reward mean: 4.5300 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4798 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.5322 +num rollout transitions: 5000, reward mean: 4.5256 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.4822 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.4771 +num rollout transitions: 5000, reward mean: 4.5196 +num rollout transitions: 5000, reward mean: 4.4934 +---------------------------------------------------------------------------------- +| alpha | 0.223 | +| eval/episode_length | 848 | +| eval/episode_length_std | 177 | +| eval/normalized_episode_reward | 89.8 | +| eval/normalized_episode_reward_std | 20.2 | +| loss/actor | -319 | +| loss/alpha | 0.00385 | +| loss/critic | 65 | +| penalty/lcb_mean | 1.85 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0.00475 | +| rollout_info/dynamics_penalty_std | 0.0659 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.4882 +num rollout transitions: 5000, reward mean: 4.5334 +num rollout transitions: 5000, reward mean: 4.5430 +num rollout transitions: 5000, reward mean: 4.5171 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5452 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5369 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.4927 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.5180 +num rollout transitions: 5000, reward mean: 4.5193 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.4817 +num rollout transitions: 5000, reward mean: 4.5298 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.4613 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.4839 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.4772 +num rollout transitions: 5000, reward mean: 4.5019 +num rollout transitions: 5000, reward mean: 4.5357 +num rollout transitions: 5000, reward mean: 4.4504 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.4567 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5203 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.4702 +---------------------------------------------------------------------------------- +| alpha | 0.22 | +| eval/episode_length | 932 | +| eval/episode_length_std | 249 | +| eval/normalized_episode_reward | 95.1 | +| eval/normalized_episode_reward_std | 26 | +| loss/actor | -319 | +| loss/alpha | -0.0215 | +| loss/critic | 67.7 | +| penalty/lcb_mean | 1.9 | +| penalty/lcb_std | 2.52 | +| rollout_info/dynamics_penalty_mean | 0.00422 | +| rollout_info/dynamics_penalty_std | 0.0626 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5011 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.5320 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4808 +num rollout transitions: 5000, reward mean: 4.4675 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.5128 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5284 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.4608 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.4661 +num rollout transitions: 5000, reward mean: 4.5128 +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.4762 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5278 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.5304 +num rollout transitions: 5000, reward mean: 4.5409 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5354 +num rollout transitions: 5000, reward mean: 4.5268 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.5146 +---------------------------------------------------------------------------------- +| alpha | 0.2 | +| eval/episode_length | 968 | +| eval/episode_length_std | 117 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 12.9 | +| loss/actor | -321 | +| loss/alpha | 0.0229 | +| loss/critic | 60.6 | +| penalty/lcb_mean | 1.81 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 0.00367 | +| rollout_info/dynamics_penalty_std | 0.0579 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5157 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.5340 +num rollout transitions: 5000, reward mean: 4.4686 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.4839 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5072 +num rollout transitions: 5000, reward mean: 4.4585 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.4734 +num rollout transitions: 5000, reward mean: 4.5255 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5065 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.4884 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.5181 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.5242 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.4580 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.5372 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.4784 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5225 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.4786 +num rollout transitions: 5000, reward mean: 4.5316 +num rollout transitions: 5000, reward mean: 4.5277 +num rollout transitions: 5000, reward mean: 4.4981 +---------------------------------------------------------------------------------- +| alpha | 0.213 | +| eval/episode_length | 483 | +| eval/episode_length_std | 406 | +| eval/normalized_episode_reward | 49 | +| eval/normalized_episode_reward_std | 47.7 | +| loss/actor | -327 | +| loss/alpha | -0.00792 | +| loss/critic | 66 | +| penalty/lcb_mean | 1.87 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0.00354 | +| rollout_info/dynamics_penalty_std | 0.0571 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.4473 +num rollout transitions: 5000, reward mean: 4.4750 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.4730 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.4789 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4648 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.4697 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4834 +num rollout transitions: 5000, reward mean: 4.4466 +num rollout transitions: 5000, reward mean: 4.4666 +num rollout transitions: 5000, reward mean: 4.5592 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.4725 +num rollout transitions: 5000, reward mean: 4.4765 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.4660 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.4684 +num rollout transitions: 5000, reward mean: 4.4680 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.4607 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4742 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.4798 +num rollout transitions: 5000, reward mean: 4.4983 +---------------------------------------------------------------------------------- +| alpha | 0.214 | +| eval/episode_length | 854 | +| eval/episode_length_std | 232 | +| eval/normalized_episode_reward | 89 | +| eval/normalized_episode_reward_std | 26.3 | +| loss/actor | -320 | +| loss/alpha | -0.0055 | +| loss/critic | 62.6 | +| penalty/lcb_mean | 1.87 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 0.0033 | +| rollout_info/dynamics_penalty_std | 0.0542 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.4863 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.5314 +num rollout transitions: 5000, reward mean: 4.5354 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.5308 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.4779 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.5240 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.5249 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.5461 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.4794 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.4790 +num rollout transitions: 5000, reward mean: 4.4634 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.4774 +num rollout transitions: 5000, reward mean: 4.4820 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.4885 +num rollout transitions: 5000, reward mean: 4.5237 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.4845 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.4869 +---------------------------------------------------------------------------------- +| alpha | 0.227 | +| eval/episode_length | 17.1 | +| eval/episode_length_std | 0.286 | +| eval/normalized_episode_reward | -0.158 | +| eval/normalized_episode_reward_std | -0.0236 | +| loss/actor | -310 | +| loss/alpha | 0.0128 | +| loss/critic | 67.4 | +| penalty/lcb_mean | 1.89 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0.00335 | +| rollout_info/dynamics_penalty_std | 0.0554 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5019 +num rollout transitions: 5000, reward mean: 4.4838 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.5247 +num rollout transitions: 5000, reward mean: 4.5106 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5416 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.4872 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.5371 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.5353 +num rollout transitions: 5000, reward mean: 4.5271 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.5248 +num rollout transitions: 5000, reward mean: 4.5511 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.5230 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.4923 +num rollout transitions: 5000, reward mean: 4.4805 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5318 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.5240 +num rollout transitions: 5000, reward mean: 4.5311 +num rollout transitions: 5000, reward mean: 4.5434 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5160 +---------------------------------------------------------------------------------- +| alpha | 0.231 | +| eval/episode_length | 55.9 | +| eval/episode_length_std | 0.51 | +| eval/normalized_episode_reward | -0.322 | +| eval/normalized_episode_reward_std | -0.0225 | +| loss/actor | -305 | +| loss/alpha | 0.00668 | +| loss/critic | 73.6 | +| penalty/lcb_mean | 2.01 | +| penalty/lcb_std | 2.52 | +| rollout_info/dynamics_penalty_mean | 0.00292 | +| rollout_info/dynamics_penalty_std | 0.0515 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.4694 +num rollout transitions: 5000, reward mean: 4.5017 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.5181 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5415 +num rollout transitions: 5000, reward mean: 4.4673 +num rollout transitions: 5000, reward mean: 4.4834 +num rollout transitions: 5000, reward mean: 4.4742 +num rollout transitions: 5000, reward mean: 4.4752 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.4787 +num rollout transitions: 5000, reward mean: 4.4814 +num rollout transitions: 5000, reward mean: 4.4632 +num rollout transitions: 5000, reward mean: 4.4583 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.5205 +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.5323 +num rollout transitions: 5000, reward mean: 4.5396 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.4930 +num rollout transitions: 5000, reward mean: 4.4834 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.4741 +num rollout transitions: 5000, reward mean: 4.4704 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.5345 +---------------------------------------------------------------------------------- +| alpha | 0.227 | +| eval/episode_length | 847 | +| eval/episode_length_std | 189 | +| eval/normalized_episode_reward | 85 | +| eval/normalized_episode_reward_std | 22.2 | +| loss/actor | -326 | +| loss/alpha | -0.0138 | +| loss/critic | 77.8 | +| penalty/lcb_mean | 2.11 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0.00283 | +| rollout_info/dynamics_penalty_std | 0.0513 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.5255 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.5378 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5400 +num rollout transitions: 5000, reward mean: 4.5295 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4844 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.4679 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.5065 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.4750 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.4880 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5154 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.4784 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.4726 +num rollout transitions: 5000, reward mean: 4.4852 +num rollout transitions: 5000, reward mean: 4.4759 +num rollout transitions: 5000, reward mean: 4.4696 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4833 +num rollout transitions: 5000, reward mean: 4.4844 +---------------------------------------------------------------------------------- +| alpha | 0.209 | +| eval/episode_length | 133 | +| eval/episode_length_std | 41.2 | +| eval/normalized_episode_reward | 2.72 | +| eval/normalized_episode_reward_std | 3.12 | +| loss/actor | -313 | +| loss/alpha | -0.00888 | +| loss/critic | 73.1 | +| penalty/lcb_mean | 2.03 | +| penalty/lcb_std | 2.68 | +| rollout_info/dynamics_penalty_mean | 0.00301 | +| rollout_info/dynamics_penalty_std | 0.0519 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4846 +num rollout transitions: 5000, reward mean: 4.4727 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.4679 +num rollout transitions: 5000, reward mean: 4.4643 +num rollout transitions: 5000, reward mean: 4.3147 +num rollout transitions: 5000, reward mean: 4.4045 +num rollout transitions: 5000, reward mean: 4.3070 +num rollout transitions: 5000, reward mean: 3.9701 +num rollout transitions: 5000, reward mean: 3.9406 +num rollout transitions: 5000, reward mean: 3.9813 +num rollout transitions: 5000, reward mean: 4.1120 +num rollout transitions: 5000, reward mean: 4.0958 +num rollout transitions: 5000, reward mean: 4.1000 +num rollout transitions: 5000, reward mean: 4.0956 +num rollout transitions: 5000, reward mean: 4.0818 +num rollout transitions: 5000, reward mean: 4.0397 +num rollout transitions: 5000, reward mean: 3.9589 +num rollout transitions: 5000, reward mean: 3.9843 +num rollout transitions: 5000, reward mean: 4.0250 +num rollout transitions: 5000, reward mean: 4.1477 +num rollout transitions: 5000, reward mean: 4.5065 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.4362 +num rollout transitions: 5000, reward mean: 4.4340 +num rollout transitions: 5000, reward mean: 4.4358 +num rollout transitions: 5000, reward mean: 4.3817 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.3966 +num rollout transitions: 5000, reward mean: 4.4790 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.4172 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.5312 +num rollout transitions: 5000, reward mean: 4.5344 +num rollout transitions: 5000, reward mean: 4.5713 +num rollout transitions: 5000, reward mean: 4.5847 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.5330 +num rollout transitions: 5000, reward mean: 4.4812 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.5411 +num rollout transitions: 5000, reward mean: 4.5342 +num rollout transitions: 5000, reward mean: 4.5298 +---------------------------------------------------------------------------------- +| alpha | 0.814 | +| eval/episode_length | 76.5 | +| eval/episode_length_std | 0.817 | +| eval/normalized_episode_reward | -0.586 | +| eval/normalized_episode_reward_std | 0.00248 | +| loss/actor | -7.8e+04 | +| loss/alpha | -34.6 | +| loss/critic | 3.81e+06 | +| penalty/lcb_mean | 488 | +| penalty/lcb_std | 449 | +| rollout_info/dynamics_penalty_mean | 0.0766 | +| rollout_info/dynamics_penalty_std | 0.229 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.37 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5758 +num rollout transitions: 5000, reward mean: 4.5262 +num rollout transitions: 5000, reward mean: 4.5275 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.5213 +num rollout transitions: 5000, reward mean: 4.4819 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.5397 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5269 +num rollout transitions: 5000, reward mean: 4.5262 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5516 +num rollout transitions: 5000, reward mean: 4.5534 +num rollout transitions: 5000, reward mean: 4.5436 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.5307 +num rollout transitions: 5000, reward mean: 4.5276 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.4787 +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.4786 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.5312 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.4696 +num rollout transitions: 5000, reward mean: 4.4744 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.4839 +num rollout transitions: 5000, reward mean: 4.5025 +----------------------------------------------------------------------------------- +| alpha | 1 | +| eval/episode_length | 81.4 | +| eval/episode_length_std | 9.06 | +| eval/normalized_episode_reward | -0.417 | +| eval/normalized_episode_reward_std | 0.102 | +| loss/actor | -2.78e+04 | +| loss/alpha | -85.4 | +| loss/critic | 5.34e+05 | +| penalty/lcb_mean | 159 | +| penalty/lcb_std | 185 | +| rollout_info/dynamics_penalty_mean | 0.00444 | +| rollout_info/dynamics_penalty_std | 0.0631 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1650000 | +----------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.5421 +num rollout transitions: 5000, reward mean: 4.4795 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.4884 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5299 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.5355 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5279 +num rollout transitions: 5000, reward mean: 4.5390 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.5524 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.5313 +num rollout transitions: 5000, reward mean: 4.5274 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5248 +num rollout transitions: 5000, reward mean: 4.5295 +num rollout transitions: 5000, reward mean: 4.5357 +num rollout transitions: 5000, reward mean: 4.5431 +num rollout transitions: 5000, reward mean: 4.5264 +num rollout transitions: 5000, reward mean: 4.5417 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.5594 +num rollout transitions: 5000, reward mean: 4.5418 +num rollout transitions: 5000, reward mean: 4.5391 +num rollout transitions: 5000, reward mean: 4.5266 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.5290 +num rollout transitions: 5000, reward mean: 4.5438 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.5013 +num rollout transitions: 5000, reward mean: 4.4883 +----------------------------------------------------------------------------------- +| alpha | 1 | +| eval/episode_length | 243 | +| eval/episode_length_std | 71.7 | +| eval/normalized_episode_reward | 8.38 | +| eval/normalized_episode_reward_std | 3.18 | +| loss/actor | -1.27e+03 | +| loss/alpha | -65.6 | +| loss/critic | 2.25e+04 | +| penalty/lcb_mean | 26.6 | +| penalty/lcb_std | 29.2 | +| rollout_info/dynamics_penalty_mean | 0.00271 | +| rollout_info/dynamics_penalty_std | 0.0501 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 1700000 | +----------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.5491 +num rollout transitions: 5000, reward mean: 4.4988 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5318 +num rollout transitions: 5000, reward mean: 4.4773 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.4839 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.5220 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.4667 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.5295 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.5175 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.4896 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.4706 +num rollout transitions: 5000, reward mean: 4.5259 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.4821 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.5368 +num rollout transitions: 5000, reward mean: 4.5346 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5399 +num rollout transitions: 5000, reward mean: 4.4788 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5277 +---------------------------------------------------------------------------------- +| alpha | 1 | +| eval/episode_length | 184 | +| eval/episode_length_std | 104 | +| eval/normalized_episode_reward | 5.78 | +| eval/normalized_episode_reward_std | 3.88 | +| loss/actor | -43.4 | +| loss/alpha | 0.674 | +| loss/critic | 484 | +| penalty/lcb_mean | 4.64 | +| penalty/lcb_std | 5.92 | +| rollout_info/dynamics_penalty_mean | 0.00235 | +| rollout_info/dynamics_penalty_std | 0.0459 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.5192 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.4896 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.4955 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.4806 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.5213 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.4833 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.4608 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.5299 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.4783 +num rollout transitions: 5000, reward mean: 4.4657 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.4798 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.5196 +num rollout transitions: 5000, reward mean: 4.4733 +num rollout transitions: 5000, reward mean: 4.4636 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.4634 +num rollout transitions: 5000, reward mean: 4.4825 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.4715 +---------------------------------------------------------------------------------- +| alpha | 1 | +| eval/episode_length | 81.7 | +| eval/episode_length_std | 22.9 | +| eval/normalized_episode_reward | 0.876 | +| eval/normalized_episode_reward_std | 0.301 | +| loss/actor | -38.3 | +| loss/alpha | 34.2 | +| loss/critic | 129 | +| penalty/lcb_mean | 2.41 | +| penalty/lcb_std | 2.87 | +| rollout_info/dynamics_penalty_mean | 0.00244 | +| rollout_info/dynamics_penalty_std | 0.0467 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.5205 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.4756 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.4728 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.4841 +num rollout transitions: 5000, reward mean: 4.4781 +num rollout transitions: 5000, reward mean: 4.4846 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4651 +num rollout transitions: 5000, reward mean: 4.4863 +num rollout transitions: 5000, reward mean: 4.4645 +num rollout transitions: 5000, reward mean: 4.4647 +num rollout transitions: 5000, reward mean: 4.4715 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.4712 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.4716 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.4769 +num rollout transitions: 5000, reward mean: 4.4652 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.4800 +num rollout transitions: 5000, reward mean: 4.4841 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.4762 +num rollout transitions: 5000, reward mean: 4.5297 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.5174 +---------------------------------------------------------------------------------- +| alpha | 1 | +| eval/episode_length | 78.4 | +| eval/episode_length_std | 4.97 | +| eval/normalized_episode_reward | -0.584 | +| eval/normalized_episode_reward_std | 0.0822 | +| loss/actor | -49.2 | +| loss/alpha | 25.6 | +| loss/critic | 66.4 | +| penalty/lcb_mean | 1.86 | +| penalty/lcb_std | 1.99 | +| rollout_info/dynamics_penalty_mean | 0.00303 | +| rollout_info/dynamics_penalty_std | 0.0524 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4790 +num rollout transitions: 5000, reward mean: 4.4803 +num rollout transitions: 5000, reward mean: 4.4852 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.4940 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.4795 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.4746 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4806 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.5308 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.5313 +num rollout transitions: 5000, reward mean: 4.4834 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5345 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.4781 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.5281 +num rollout transitions: 5000, reward mean: 4.5370 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.5486 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.5362 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.5303 +---------------------------------------------------------------------------------- +| alpha | 0.634 | +| eval/episode_length | 72.5 | +| eval/episode_length_std | 17.2 | +| eval/normalized_episode_reward | 1.13 | +| eval/normalized_episode_reward_std | 0.742 | +| loss/actor | -109 | +| loss/alpha | 1.84 | +| loss/critic | 57.9 | +| penalty/lcb_mean | 1.74 | +| penalty/lcb_std | 1.77 | +| rollout_info/dynamics_penalty_mean | 0.00304 | +| rollout_info/dynamics_penalty_std | 0.0529 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5345 +num rollout transitions: 5000, reward mean: 4.5508 +num rollout transitions: 5000, reward mean: 4.5250 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.4829 +num rollout transitions: 5000, reward mean: 4.5259 +num rollout transitions: 5000, reward mean: 4.4743 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.4736 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5318 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.5544 +num rollout transitions: 5000, reward mean: 4.5322 +num rollout transitions: 5000, reward mean: 4.5374 +num rollout transitions: 5000, reward mean: 4.5193 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.5192 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5573 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.4824 +num rollout transitions: 5000, reward mean: 4.5457 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.4743 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.5277 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.5393 +num rollout transitions: 5000, reward mean: 4.5321 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5333 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.5357 +num rollout transitions: 5000, reward mean: 4.5281 +num rollout transitions: 5000, reward mean: 4.5345 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.4958 +---------------------------------------------------------------------------------- +| alpha | 0.267 | +| eval/episode_length | 17 | +| eval/episode_length_std | 0.49 | +| eval/normalized_episode_reward | 0.044 | +| eval/normalized_episode_reward_std | -0.025 | +| loss/actor | -214 | +| loss/alpha | 0.0147 | +| loss/critic | 85.1 | +| penalty/lcb_mean | 2.11 | +| penalty/lcb_std | 2.16 | +| rollout_info/dynamics_penalty_mean | 0.00506 | +| rollout_info/dynamics_penalty_std | 0.0662 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.5436 +num rollout transitions: 5000, reward mean: 4.5545 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.5433 +num rollout transitions: 5000, reward mean: 4.5021 +num rollout transitions: 5000, reward mean: 4.5538 +num rollout transitions: 5000, reward mean: 4.5293 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5307 +num rollout transitions: 5000, reward mean: 4.5422 +num rollout transitions: 5000, reward mean: 4.5533 +num rollout transitions: 5000, reward mean: 4.5629 +num rollout transitions: 5000, reward mean: 4.5287 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.5165 +num rollout transitions: 5000, reward mean: 4.5157 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.5096 +num rollout transitions: 5000, reward mean: 4.5252 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5154 +num rollout transitions: 5000, reward mean: 4.5194 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.5194 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5170 +num rollout transitions: 5000, reward mean: 4.5254 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.5217 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.4865 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5134 +---------------------------------------------------------------------------------- +| alpha | 0.273 | +| eval/episode_length | 47.5 | +| eval/episode_length_std | 24.4 | +| eval/normalized_episode_reward | -0.154 | +| eval/normalized_episode_reward_std | 0.229 | +| loss/actor | -217 | +| loss/alpha | -0.0056 | +| loss/critic | 76.4 | +| penalty/lcb_mean | 1.99 | +| penalty/lcb_std | 2.09 | +| rollout_info/dynamics_penalty_mean | 0.00361 | +| rollout_info/dynamics_penalty_std | 0.0564 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.4732 +num rollout transitions: 5000, reward mean: 4.5619 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.5263 +num rollout transitions: 5000, reward mean: 4.5361 +num rollout transitions: 5000, reward mean: 4.5248 +num rollout transitions: 5000, reward mean: 4.5350 +num rollout transitions: 5000, reward mean: 4.4711 +num rollout transitions: 5000, reward mean: 4.4671 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.4802 +num rollout transitions: 5000, reward mean: 4.5106 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.4618 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.5350 +num rollout transitions: 5000, reward mean: 4.4629 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.4671 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.5290 +num rollout transitions: 5000, reward mean: 4.5209 +---------------------------------------------------------------------------------- +| alpha | 0.265 | +| eval/episode_length | 22 | +| eval/episode_length_std | 19.6 | +| eval/normalized_episode_reward | 0.124 | +| eval/normalized_episode_reward_std | 0.691 | +| loss/actor | -215 | +| loss/alpha | 0.0016 | +| loss/critic | 69.3 | +| penalty/lcb_mean | 1.91 | +| penalty/lcb_std | 1.95 | +| rollout_info/dynamics_penalty_mean | 0.00204 | +| rollout_info/dynamics_penalty_std | 0.0421 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.5285 +num rollout transitions: 5000, reward mean: 4.5222 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.4736 +num rollout transitions: 5000, reward mean: 4.4865 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.5174 +num rollout transitions: 5000, reward mean: 4.5427 +num rollout transitions: 5000, reward mean: 4.5489 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5363 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.5278 +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.5338 +num rollout transitions: 5000, reward mean: 4.5314 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.4863 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5283 +num rollout transitions: 5000, reward mean: 4.5180 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5327 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.4509 +---------------------------------------------------------------------------------- +| alpha | 0.267 | +| eval/episode_length | 80 | +| eval/episode_length_std | 9.24 | +| eval/normalized_episode_reward | 3.49 | +| eval/normalized_episode_reward_std | 0.849 | +| loss/actor | -236 | +| loss/alpha | -0.00345 | +| loss/critic | 67.2 | +| penalty/lcb_mean | 1.89 | +| penalty/lcb_std | 1.95 | +| rollout_info/dynamics_penalty_mean | 0.00373 | +| rollout_info/dynamics_penalty_std | 0.0576 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.4825 +num rollout transitions: 5000, reward mean: 4.5269 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.5174 +num rollout transitions: 5000, reward mean: 4.5291 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.5174 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.4863 +num rollout transitions: 5000, reward mean: 4.4803 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.4613 +num rollout transitions: 5000, reward mean: 4.4770 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.5275 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.4563 +num rollout transitions: 5000, reward mean: 4.5011 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5367 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5329 +num rollout transitions: 5000, reward mean: 4.5328 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.4709 +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.4818 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.4708 +num rollout transitions: 5000, reward mean: 4.4709 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.5472 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.5237 +---------------------------------------------------------------------------------- +| alpha | 0.255 | +| eval/episode_length | 38.8 | +| eval/episode_length_std | 34.6 | +| eval/normalized_episode_reward | 0.684 | +| eval/normalized_episode_reward_std | 1.99 | +| loss/actor | -240 | +| loss/alpha | 0.00471 | +| loss/critic | 66.2 | +| penalty/lcb_mean | 1.92 | +| penalty/lcb_std | 1.94 | +| rollout_info/dynamics_penalty_mean | 0.00482 | +| rollout_info/dynamics_penalty_std | 0.0643 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.4778 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.4746 +num rollout transitions: 5000, reward mean: 4.4829 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.5548 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.4798 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.4815 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4605 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.4680 +num rollout transitions: 5000, reward mean: 4.4904 +num rollout transitions: 5000, reward mean: 4.4700 +num rollout transitions: 5000, reward mean: 4.5422 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.4733 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5360 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.4838 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4949 +num rollout transitions: 5000, reward mean: 4.5365 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5407 +---------------------------------------------------------------------------------- +| alpha | 0.245 | +| eval/episode_length | 19 | +| eval/episode_length_std | 6.55 | +| eval/normalized_episode_reward | 0.0265 | +| eval/normalized_episode_reward_std | -0.0205 | +| loss/actor | -244 | +| loss/alpha | -0.0114 | +| loss/critic | 61 | +| penalty/lcb_mean | 1.81 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 0.00244 | +| rollout_info/dynamics_penalty_std | 0.046 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.4825 +num rollout transitions: 5000, reward mean: 4.5263 +num rollout transitions: 5000, reward mean: 4.5279 +num rollout transitions: 5000, reward mean: 4.5188 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.4825 +num rollout transitions: 5000, reward mean: 4.4872 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.5441 +num rollout transitions: 5000, reward mean: 4.4763 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5334 +num rollout transitions: 5000, reward mean: 4.4865 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.4768 +num rollout transitions: 5000, reward mean: 4.4717 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.4770 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5296 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.5351 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.5248 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5242 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5067 +---------------------------------------------------------------------------------- +| alpha | 0.239 | +| eval/episode_length | 49.7 | +| eval/episode_length_std | 11.1 | +| eval/normalized_episode_reward | -0.235 | +| eval/normalized_episode_reward_std | 0.0427 | +| loss/actor | -250 | +| loss/alpha | -0.00436 | +| loss/critic | 65.8 | +| penalty/lcb_mean | 1.88 | +| penalty/lcb_std | 2.02 | +| rollout_info/dynamics_penalty_mean | 0.00209 | +| rollout_info/dynamics_penalty_std | 0.0436 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.4723 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5368 +num rollout transitions: 5000, reward mean: 4.5080 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.4772 +num rollout transitions: 5000, reward mean: 4.4694 +num rollout transitions: 5000, reward mean: 4.5368 +num rollout transitions: 5000, reward mean: 4.4732 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.4796 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.5339 +num rollout transitions: 5000, reward mean: 4.4778 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.4805 +num rollout transitions: 5000, reward mean: 4.5287 +num rollout transitions: 5000, reward mean: 4.4697 +num rollout transitions: 5000, reward mean: 4.4848 +num rollout transitions: 5000, reward mean: 4.4819 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5381 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.5529 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.5188 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.5128 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.4734 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4998 +---------------------------------------------------------------------------------- +| alpha | 0.229 | +| eval/episode_length | 260 | +| eval/episode_length_std | 234 | +| eval/normalized_episode_reward | 10 | +| eval/normalized_episode_reward_std | 16.4 | +| loss/actor | -249 | +| loss/alpha | 0.00654 | +| loss/critic | 64.3 | +| penalty/lcb_mean | 1.84 | +| penalty/lcb_std | 1.99 | +| rollout_info/dynamics_penalty_mean | 0.00226 | +| rollout_info/dynamics_penalty_std | 0.0448 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5205 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5376 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5162 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.4716 +num rollout transitions: 5000, reward mean: 4.5286 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.5071 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.4828 +num rollout transitions: 5000, reward mean: 4.4619 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.4753 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.5194 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.4940 +num rollout transitions: 5000, reward mean: 4.4706 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.4583 +num rollout transitions: 5000, reward mean: 4.4812 +num rollout transitions: 5000, reward mean: 4.4781 +num rollout transitions: 5000, reward mean: 4.4851 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.4772 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.4923 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.4833 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.5021 +num rollout transitions: 5000, reward mean: 4.4812 +num rollout transitions: 5000, reward mean: 4.5064 +---------------------------------------------------------------------------------- +| alpha | 0.213 | +| eval/episode_length | 240 | +| eval/episode_length_std | 182 | +| eval/normalized_episode_reward | 14.8 | +| eval/normalized_episode_reward_std | 17.7 | +| loss/actor | -252 | +| loss/alpha | -0.00163 | +| loss/critic | 58 | +| penalty/lcb_mean | 1.74 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 0.00265 | +| rollout_info/dynamics_penalty_std | 0.0487 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4660 +num rollout transitions: 5000, reward mean: 4.4783 +num rollout transitions: 5000, reward mean: 4.4595 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.5312 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.4519 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.4669 +num rollout transitions: 5000, reward mean: 4.5021 +num rollout transitions: 5000, reward mean: 4.4832 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.5315 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.5291 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.4659 +num rollout transitions: 5000, reward mean: 4.4580 +num rollout transitions: 5000, reward mean: 4.4709 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.4851 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.5262 +num rollout transitions: 5000, reward mean: 4.4795 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.4620 +num rollout transitions: 5000, reward mean: 4.4927 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.4664 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.4799 +num rollout transitions: 5000, reward mean: 4.5171 +num rollout transitions: 5000, reward mean: 4.5251 +---------------------------------------------------------------------------------- +| alpha | 0.209 | +| eval/episode_length | 154 | +| eval/episode_length_std | 12.4 | +| eval/normalized_episode_reward | 7.27 | +| eval/normalized_episode_reward_std | 1.09 | +| loss/actor | -251 | +| loss/alpha | -0.00813 | +| loss/critic | 56.1 | +| penalty/lcb_mean | 1.74 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 0.00209 | +| rollout_info/dynamics_penalty_std | 0.0433 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4769 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5410 +num rollout transitions: 5000, reward mean: 4.5345 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.5265 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.4804 +num rollout transitions: 5000, reward mean: 4.4800 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.4803 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5301 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.5253 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.5313 +num rollout transitions: 5000, reward mean: 4.4816 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.4603 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5011 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.4774 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.4879 +num rollout transitions: 5000, reward mean: 4.4822 +num rollout transitions: 5000, reward mean: 4.4754 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.4808 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.4988 +num rollout transitions: 5000, reward mean: 4.5018 +---------------------------------------------------------------------------------- +| alpha | 0.209 | +| eval/episode_length | 418 | +| eval/episode_length_std | 348 | +| eval/normalized_episode_reward | 29.4 | +| eval/normalized_episode_reward_std | 39.2 | +| loss/actor | -246 | +| loss/alpha | 0.00635 | +| loss/critic | 59.5 | +| penalty/lcb_mean | 1.76 | +| penalty/lcb_std | 1.96 | +| rollout_info/dynamics_penalty_mean | 0.00231 | +| rollout_info/dynamics_penalty_std | 0.0457 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.4925 +num rollout transitions: 5000, reward mean: 4.4872 +num rollout transitions: 5000, reward mean: 4.5065 +num rollout transitions: 5000, reward mean: 4.4783 +num rollout transitions: 5000, reward mean: 4.4778 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.5401 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.5307 +num rollout transitions: 5000, reward mean: 4.4695 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.4759 +num rollout transitions: 5000, reward mean: 4.4872 +num rollout transitions: 5000, reward mean: 4.5567 +num rollout transitions: 5000, reward mean: 4.5483 +num rollout transitions: 5000, reward mean: 4.5260 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.4664 +num rollout transitions: 5000, reward mean: 4.4683 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.4615 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.4848 +num rollout transitions: 5000, reward mean: 4.5309 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.5019 +num rollout transitions: 5000, reward mean: 4.5245 +num rollout transitions: 5000, reward mean: 4.4882 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.5241 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4833 +num rollout transitions: 5000, reward mean: 4.4777 +num rollout transitions: 5000, reward mean: 4.4971 +---------------------------------------------------------------------------------- +| alpha | 0.21 | +| eval/episode_length | 585 | +| eval/episode_length_std | 396 | +| eval/normalized_episode_reward | 55.2 | +| eval/normalized_episode_reward_std | 42.4 | +| loss/actor | -249 | +| loss/alpha | -0.0122 | +| loss/critic | 57 | +| penalty/lcb_mean | 1.75 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 0.00226 | +| rollout_info/dynamics_penalty_std | 0.0453 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.4457 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.4718 +num rollout transitions: 5000, reward mean: 4.4878 +num rollout transitions: 5000, reward mean: 4.4868 +num rollout transitions: 5000, reward mean: 4.5341 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.5225 +num rollout transitions: 5000, reward mean: 4.4885 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5178 +num rollout transitions: 5000, reward mean: 4.5242 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.4852 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.5175 +num rollout transitions: 5000, reward mean: 4.5369 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.4778 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.4791 +num rollout transitions: 5000, reward mean: 4.5364 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.5287 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.5178 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.4773 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.4956 +---------------------------------------------------------------------------------- +| alpha | 0.195 | +| eval/episode_length | 847 | +| eval/episode_length_std | 317 | +| eval/normalized_episode_reward | 88.6 | +| eval/normalized_episode_reward_std | 36.5 | +| loss/actor | -260 | +| loss/alpha | 0.0033 | +| loss/critic | 53.9 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 1.84 | +| rollout_info/dynamics_penalty_mean | 0.00172 | +| rollout_info/dynamics_penalty_std | 0.0396 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.4781 +num rollout transitions: 5000, reward mean: 4.4885 +num rollout transitions: 5000, reward mean: 4.4646 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.4839 +num rollout transitions: 5000, reward mean: 4.5238 +num rollout transitions: 5000, reward mean: 4.4890 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.4758 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.4786 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.4779 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.5128 +num rollout transitions: 5000, reward mean: 4.4963 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.4802 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5326 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.5106 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.5300 +num rollout transitions: 5000, reward mean: 4.4846 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5102 +num rollout transitions: 5000, reward mean: 4.5168 +---------------------------------------------------------------------------------- +| alpha | 0.197 | +| eval/episode_length | 649 | +| eval/episode_length_std | 373 | +| eval/normalized_episode_reward | 11.7 | +| eval/normalized_episode_reward_std | 8.53 | +| loss/actor | -260 | +| loss/alpha | 0.000623 | +| loss/critic | 55.1 | +| penalty/lcb_mean | 1.73 | +| penalty/lcb_std | 1.85 | +| rollout_info/dynamics_penalty_mean | 0.00254 | +| rollout_info/dynamics_penalty_std | 0.0476 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5266 +num rollout transitions: 5000, reward mean: 4.5406 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.4777 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.4696 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.4735 +num rollout transitions: 5000, reward mean: 4.5283 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.5178 +num rollout transitions: 5000, reward mean: 4.5230 +num rollout transitions: 5000, reward mean: 4.5242 +num rollout transitions: 5000, reward mean: 4.5474 +num rollout transitions: 5000, reward mean: 4.5350 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4722 +num rollout transitions: 5000, reward mean: 4.5196 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.5165 +num rollout transitions: 5000, reward mean: 4.5106 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.5313 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5197 +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.4798 +num rollout transitions: 5000, reward mean: 4.4719 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.5286 +num rollout transitions: 5000, reward mean: 4.5380 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.4670 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4733 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.4634 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.4835 +---------------------------------------------------------------------------------- +| alpha | 0.208 | +| eval/episode_length | 922 | +| eval/episode_length_std | 132 | +| eval/normalized_episode_reward | 94.7 | +| eval/normalized_episode_reward_std | 15.3 | +| loss/actor | -267 | +| loss/alpha | 0.00797 | +| loss/critic | 58 | +| penalty/lcb_mean | 1.78 | +| penalty/lcb_std | 1.92 | +| rollout_info/dynamics_penalty_mean | 0.00312 | +| rollout_info/dynamics_penalty_std | 0.054 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5634 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.5354 +num rollout transitions: 5000, reward mean: 4.5389 +num rollout transitions: 5000, reward mean: 4.5252 +num rollout transitions: 5000, reward mean: 4.5279 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5439 +num rollout transitions: 5000, reward mean: 4.5305 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.5071 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.4803 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5240 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.4725 +num rollout transitions: 5000, reward mean: 4.4763 +num rollout transitions: 5000, reward mean: 4.4753 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.5217 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.4829 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5254 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.4805 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.5039 +---------------------------------------------------------------------------------- +| alpha | 0.208 | +| eval/episode_length | 358 | +| eval/episode_length_std | 332 | +| eval/normalized_episode_reward | 34.4 | +| eval/normalized_episode_reward_std | 39.6 | +| loss/actor | -263 | +| loss/alpha | -0.0143 | +| loss/critic | 58.6 | +| penalty/lcb_mean | 1.79 | +| penalty/lcb_std | 1.93 | +| rollout_info/dynamics_penalty_mean | 0.00279 | +| rollout_info/dynamics_penalty_std | 0.051 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4706 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.5355 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.4618 +num rollout transitions: 5000, reward mean: 4.4832 +num rollout transitions: 5000, reward mean: 4.4755 +num rollout transitions: 5000, reward mean: 4.4774 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.4734 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5309 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.5257 +num rollout transitions: 5000, reward mean: 4.4732 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.4643 +num rollout transitions: 5000, reward mean: 4.4719 +num rollout transitions: 5000, reward mean: 4.4851 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.4650 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4657 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.4487 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.4351 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.4683 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.4811 +---------------------------------------------------------------------------------- +| alpha | 0.207 | +| eval/episode_length | 212 | +| eval/episode_length_std | 10.5 | +| eval/normalized_episode_reward | 14.4 | +| eval/normalized_episode_reward_std | 1.07 | +| loss/actor | -261 | +| loss/alpha | 0.0168 | +| loss/critic | 65.9 | +| penalty/lcb_mean | 1.9 | +| penalty/lcb_std | 2.11 | +| rollout_info/dynamics_penalty_mean | 0.00318 | +| rollout_info/dynamics_penalty_std | 0.0545 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.5364 +num rollout transitions: 5000, reward mean: 4.5337 +num rollout transitions: 5000, reward mean: 4.4840 +num rollout transitions: 5000, reward mean: 4.4955 +num rollout transitions: 5000, reward mean: 4.4820 +num rollout transitions: 5000, reward mean: 4.5230 +num rollout transitions: 5000, reward mean: 4.4669 +num rollout transitions: 5000, reward mean: 4.4747 +num rollout transitions: 5000, reward mean: 4.4760 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.4735 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.4844 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4779 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5240 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.4963 +num rollout transitions: 5000, reward mean: 4.4863 +num rollout transitions: 5000, reward mean: 4.4902 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.4820 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.4691 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.4872 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.4841 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.4510 +num rollout transitions: 5000, reward mean: 4.4536 +---------------------------------------------------------------------------------- +| alpha | 0.198 | +| eval/episode_length | 994 | +| eval/episode_length_std | 56.2 | +| eval/normalized_episode_reward | 98.9 | +| eval/normalized_episode_reward_std | 6.47 | +| loss/actor | -257 | +| loss/alpha | 0.00315 | +| loss/critic | 70.6 | +| penalty/lcb_mean | 1.96 | +| penalty/lcb_std | 2.17 | +| rollout_info/dynamics_penalty_mean | 0.00267 | +| rollout_info/dynamics_penalty_std | 0.0499 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.4745 +num rollout transitions: 5000, reward mean: 4.4728 +num rollout transitions: 5000, reward mean: 4.4818 +num rollout transitions: 5000, reward mean: 4.4749 +num rollout transitions: 5000, reward mean: 4.4659 +num rollout transitions: 5000, reward mean: 4.4757 +num rollout transitions: 5000, reward mean: 4.4668 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.5315 +num rollout transitions: 5000, reward mean: 4.4778 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.4564 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4844 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.4667 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.4617 +num rollout transitions: 5000, reward mean: 4.4647 +num rollout transitions: 5000, reward mean: 4.4714 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.5019 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.4630 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.5072 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.4765 +num rollout transitions: 5000, reward mean: 4.4607 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5246 +---------------------------------------------------------------------------------- +| alpha | 0.195 | +| eval/episode_length | 908 | +| eval/episode_length_std | 191 | +| eval/normalized_episode_reward | 87.2 | +| eval/normalized_episode_reward_std | 23.5 | +| loss/actor | -248 | +| loss/alpha | 0.00984 | +| loss/critic | 68.7 | +| penalty/lcb_mean | 1.95 | +| penalty/lcb_std | 2.12 | +| rollout_info/dynamics_penalty_mean | 0.00282 | +| rollout_info/dynamics_penalty_std | 0.0514 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4605 +num rollout transitions: 5000, reward mean: 4.4896 +num rollout transitions: 5000, reward mean: 4.4761 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.4842 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.4930 +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.4680 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.4694 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.4804 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.4685 +num rollout transitions: 5000, reward mean: 4.4587 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.4858 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.4781 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4776 +num rollout transitions: 5000, reward mean: 4.4692 +num rollout transitions: 5000, reward mean: 4.4712 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.4667 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.4894 +---------------------------------------------------------------------------------- +| alpha | 0.193 | +| eval/episode_length | 193 | +| eval/episode_length_std | 2.49 | +| eval/normalized_episode_reward | 10.1 | +| eval/normalized_episode_reward_std | 0.17 | +| loss/actor | -248 | +| loss/alpha | -0.00323 | +| loss/critic | 71.2 | +| penalty/lcb_mean | 1.99 | +| penalty/lcb_std | 2.17 | +| rollout_info/dynamics_penalty_mean | 0.00357 | +| rollout_info/dynamics_penalty_std | 0.0579 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.5247 +num rollout transitions: 5000, reward mean: 4.4761 +num rollout transitions: 5000, reward mean: 4.4782 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.4526 +num rollout transitions: 5000, reward mean: 4.5258 +num rollout transitions: 5000, reward mean: 4.4795 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.4768 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.4682 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.4829 +num rollout transitions: 5000, reward mean: 4.4778 +num rollout transitions: 5000, reward mean: 4.4777 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.4988 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.4774 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4776 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.4890 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.5205 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.5162 +num rollout transitions: 5000, reward mean: 4.4604 +num rollout transitions: 5000, reward mean: 4.4702 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.5109 +---------------------------------------------------------------------------------- +| alpha | 0.193 | +| eval/episode_length | 171 | +| eval/episode_length_std | 83.9 | +| eval/normalized_episode_reward | 9.72 | +| eval/normalized_episode_reward_std | 9.25 | +| loss/actor | -255 | +| loss/alpha | -0.0145 | +| loss/critic | 73.6 | +| penalty/lcb_mean | 2.07 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 0.00369 | +| rollout_info/dynamics_penalty_std | 0.0591 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.4797 +num rollout transitions: 5000, reward mean: 4.4702 +num rollout transitions: 5000, reward mean: 4.5165 +num rollout transitions: 5000, reward mean: 4.4840 +num rollout transitions: 5000, reward mean: 4.5096 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.5302 +num rollout transitions: 5000, reward mean: 4.5180 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.4868 +num rollout transitions: 5000, reward mean: 4.4742 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.5264 +num rollout transitions: 5000, reward mean: 4.4753 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.4659 +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.4552 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5326 +num rollout transitions: 5000, reward mean: 4.5262 +num rollout transitions: 5000, reward mean: 4.4868 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.5420 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5446 +num rollout transitions: 5000, reward mean: 4.5193 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.4808 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.4816 +---------------------------------------------------------------------------------- +| alpha | 0.203 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 0.962 | +| loss/actor | -235 | +| loss/alpha | 0.0465 | +| loss/critic | 72.4 | +| penalty/lcb_mean | 1.99 | +| penalty/lcb_std | 2.2 | +| rollout_info/dynamics_penalty_mean | 0.00372 | +| rollout_info/dynamics_penalty_std | 0.0594 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 57210.08s diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_42×tamp_26-0506-193338/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_42×tamp_26-0506-193338/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_42×tamp_26-0506-193338/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_42×tamp_26-0506-193338/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..1a73e5d44dd22665290e33421405b611ae31586a --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_42×tamp_26-0506-193338/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_neuralode.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 42, "device": "cuda:4", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_42×tamp_26-0112-030600/model", "reward_penalty_coef": 0.3, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/neuralODE", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 5000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "neuralODE", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "neuralODE"} diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_42×tamp_26-0506-193338/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_42×tamp_26-0506-193338/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..5c7009e23a7f6194d2cc63924368a6c921fa9646 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_42×tamp_26-0506-193338/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.3058952808371202,113.69,14.961747892542501,0.8263665595857084,0.30010501725200023,-303.5285283537049,-1.5606456364933945,27.82261092248884,1.2233473162002146,1.4966837593168503,0.01746100368283619,0.10720243709739508,5000.0,4.5199378627235305,50000 +0.15267385424584373,190.9,153.7806554804602,4.079036766502375,3.935717876120933,-421.56214651306004,-0.018222384244575993,29.721998625583588,1.2528676581847713,1.7802928282546908,0.003920576845876872,0.056658356671296144,5000.0,4.518147450638557,100000 +0.15690761288792018,95.99,3.365991681510814,4.207171659891474,0.13060915079209254,-417.1575239056414,-0.003569371287114844,35.278548392352505,1.3748821476602562,1.9798800926637814,0.005355660978776402,0.06821775135184395,5000.0,4.509379943583599,150000 +0.15790422109603766,98.75,1.5960889699512368,4.368778438991764,0.012604743002728252,-409.01877583984293,0.026509945099614504,36.89859667690328,1.427275607210403,1.9839837678003305,0.006118523312006144,0.07345865958598105,5000.0,4.506620930392868,200000 +0.18269778200924208,185.08,4.291107083259517,8.42407143025737,0.1366578062781375,-390.9472462182636,0.003356299589481264,41.72871569662143,1.4802711899340104,2.0826721090781635,0.005206996621854607,0.06902712062244033,5000.0,4.518207833807264,250000 +0.1784942957320843,164.57,18.371311874768224,11.213942413139803,1.8122253673671453,-379.0237641735865,-0.007454578464068561,43.04291785245975,1.539635018060204,2.0728883769130713,0.0027487797769913448,0.049235936442037616,5000.0,4.5154022306834944,300000 +0.18554104763299256,118.33,2.6535071132371213,5.654256476734103,0.13502500315406418,-371.99776588806236,-0.000339618375413282,41.56001968036635,1.4962046158742839,2.0807677646303064,0.002611668660074473,0.04877349542098987,5000.0,4.510903309679496,350000 +0.1792455724081416,269.61,266.27921041643486,23.68369752749192,30.207342530028413,-370.8400169500722,0.014345005304440866,41.45235708459846,1.4965416585993867,2.0652364190518906,0.003142284338174855,0.053138603807960115,5000.0,4.5107287582423705,400000 +0.19542263552546424,923.41,189.49417378906404,90.44363944345653,19.906086130764848,-368.96602858398273,-0.0011174838020279428,48.82340838500884,1.6036444358706368,2.2771869656491397,0.003431060977277347,0.055494127043676504,5000.0,4.515048228167636,450000 +0.19449299195945122,124.67,2.078725571113224,7.075877628682774,0.10415560938245613,-366.73703634398635,0.0019007465508208028,51.71688984916675,1.6672545528936193,2.371310188789317,0.006610647328299469,0.07652226714094602,5000.0,4.497770582249332,500000 +0.18667591846555506,121.9,3.0446674695276656,6.681190225967024,0.2592467996967588,-357.9242716625952,-0.008368718860484944,48.494794524039776,1.5596070430564994,2.275730357139117,0.005329465844191608,0.069706896725669,5000.0,4.505035610708929,550000 +0.20501373004823661,984.31,109.83111535443864,97.61097383736234,11.922422938354483,-346.60074414184896,0.012636555143557777,53.753061335182075,1.6697218951082045,2.363973913671969,0.004594988442595118,0.06516118682125831,5000.0,4.5043282401374,600000 +0.21204287224501214,660.3,252.36808435299423,67.20268780798087,28.186766470746605,-337.16209508849636,-9.088148426264021e-05,55.228335752144304,1.6648855627584433,2.3517726922560507,0.005170970071292484,0.0696311491017252,5000.0,4.500219200724167,650000 +0.1923804713892917,123.57,12.2452072256863,6.821454641836739,1.1822713902114586,-341.5269587780788,-0.008412675267979564,51.541011030197375,1.5896819409894776,2.226189879822673,0.007768660154687705,0.08457874696702102,5000.0,4.501755014562536,700000 +0.18846681520372757,164.59,8.636081287250601,10.516957347153086,0.7007494922087921,-339.9165009814531,0.007127368414774476,54.36619017909988,1.6869844228029272,2.3173348328042223,0.00634430078104895,0.07592773971853016,5000.0,4.506436839150316,750000 +0.21374615857899493,978.02,119.11515268848039,94.14019858053774,12.682156313708589,-337.36691073486844,-1.0814254060246899e-05,58.04062786148143,1.7643825062179586,2.3086038016462096,0.003870883145360276,0.05878065508992932,5000.0,4.507402388764968,800000 +0.20145427014827708,495.63,417.35945310966656,45.55128137583383,42.90131611217579,-331.4477695428432,-0.016949251813851113,58.55988303150108,1.7698586083483765,2.3774058134197977,0.005977843025693671,0.07315058033655225,5000.0,4.500469789328899,850000 +0.2037131784275158,526.35,235.3045420301104,52.19347058730761,27.41709909355975,-328.0005721551464,0.020625681838597587,60.1938774733355,1.7860177802252604,2.371793157005281,0.0050334465190703045,0.06768230776189264,5000.0,4.498799517040513,900000 +0.2240610484525572,905.73,171.1370126536045,91.48814927452753,18.794006178570346,-321.14623786987687,-0.0024638706550932287,71.09410331161536,1.9186732294511857,2.5305403283643866,0.0031154854270368823,0.05270814022500278,5000.0,4.501858817731638,950000 +0.21192943045080015,245.46,320.2406726198282,20.672353960114183,34.33710173962045,-318.3372531982437,0.004277628838829564,60.11424601360281,1.7756836157608007,2.383834949572085,0.004071644651185721,0.060850391325301165,5000.0,4.494767044942252,1000000 +0.21401226415276328,1000.0,0.0,103.03410551622565,1.1073274386402896,-325.6151335864255,-0.007120521165728469,57.93273035202025,1.7616096347808916,2.3081429673659204,0.0040761009501405045,0.06186791109893136,5000.0,4.498203140007965,1050000 +0.21440788798510807,1000.0,0.0,106.17626399817863,0.8146814531065395,-322.6665483715818,0.008309907034374831,59.21681589572912,1.7887043197703398,2.339434082965865,0.005038192235326861,0.06878971201432245,5000.0,4.500114455277007,1100000 +0.22305930479109348,848.45,177.35503235036776,89.78664298719006,20.2303904027085,-318.92826038391246,0.0038520461761205995,64.9781769808961,1.8521693754625372,2.4533122401761918,0.004752223461683838,0.06593292431094978,5000.0,4.498178915130432,1150000 +0.2199345981869107,931.77,248.69659647852038,95.12213637936772,26.030597018884922,-319.0141187640417,-0.021481898799166347,67.65013181675121,1.897425329496861,2.5206317217493286,0.004217508226907346,0.0625984577522344,5000.0,4.503397461659309,1200000 +0.19980159907430126,967.75,117.07043819854779,101.00750421736511,12.900474828885658,-321.46332122314226,0.022875687422603173,60.552026669808065,1.8108864265608702,2.3835231781411403,0.0036685421092640604,0.057863041152478506,5000.0,4.503823063024044,1250000 +0.21309406836360661,482.64,405.5506508440099,48.95470032364928,47.66007268442379,-326.6877467730722,-0.007915095589869026,65.9846363759225,1.8710099623966432,2.4603604353904474,0.0035423159918258895,0.057079320962537726,5000.0,4.502321241767219,1300000 +0.21406957287013506,854.06,231.60012176162604,88.95828553973885,26.317022360798138,-319.643136063846,-0.005503753688242284,62.61043613536827,1.8671378959798712,2.3957366677761183,0.00329928506650403,0.05417620149643494,5000.0,4.493024906132394,1350000 +0.22663668817132918,17.09,0.28618176042508375,-0.1579017912876373,-0.023649820409186898,-309.7080717370603,0.012761841097455639,67.44727879138831,1.8904945752477618,2.492395362005197,0.0033499486932014127,0.055412747297289885,5000.0,4.4989495172674125,1400000 +0.23117915878176504,55.86,0.5102940328869229,-0.321750487777593,-0.02246502413246179,-304.8218816741883,0.00667536309184507,73.58030368656152,2.009147656307214,2.5213319498586584,0.0029216307206689387,0.051495670573226834,5000.0,4.509956122082508,1450000 +0.2271064797341825,847.14,189.01878319362868,85.0389198705175,22.183200162968383,-326.08660251098127,-0.013786282621342842,77.84075254062756,2.1139820250057806,2.648775400493161,0.0028277267630361035,0.05127126329896476,5000.0,4.498545250902572,1500000 +0.20879124977588503,132.78,41.21688488956923,2.7210144727085788,3.1158782620182066,-313.342051655279,-0.008881910029817343,73.13480290279323,2.0302041515350338,2.682789878976292,0.003011259790196084,0.05191729241000533,5000.0,4.500417997765766,1550000 +0.813841833963098,76.45,0.8170067319184094,-0.5855106594824939,0.0024758524996155466,-78039.19664312944,-34.57848603019064,3808858.9051337284,488.3480779656322,449.09829753944456,0.07662391021611006,0.22900404926537207,5000.0,4.366522972062993,1600000 +1.0,81.42,9.055583912702703,-0.4172457148167341,0.10157073369893796,-27791.652800839445,-85.38009092178419,534011.0999734496,159.03828478431743,184.69899980613818,0.004441664051637049,0.06307298225710062,5000.0,4.5119468646809,1650000 +1.0,242.99,71.69999930264993,8.378942570826355,3.1759656797540092,-1268.495785175469,-65.59077854221417,22469.270634220637,26.571826716375416,29.208991657686095,0.0027132333665210754,0.050109224530588976,5000.0,4.518422296420088,1700000 +1.0,184.2,103.72550313206486,5.778922560038213,3.876909592546233,-43.39165672426249,0.6735391816262289,483.881286963039,4.641645192995171,5.917889915332879,0.002350891297954134,0.045854967231960427,5000.0,4.505836564451785,1750000 +1.0,81.73,22.873502136752037,0.8763171462675132,0.30072223959168143,-38.26797955116247,34.161735281715316,129.34562538330076,2.410512821063986,2.865471224944621,0.0024434961431548004,0.04669713568393024,5000.0,4.497503033863465,1800000 +1.0,78.38,4.971478653278117,-0.5838127956983499,0.08215184621636883,-49.154549486007596,25.61507783840154,66.35255301811192,1.859224475364685,1.9916172569370156,0.00303438869662024,0.052368517591407,5000.0,4.4909539864159,1850000 +0.6338073385825734,72.46,17.24727224809767,1.125388356203813,0.7416170667732822,-108.72904950859014,1.8424794612988102,57.87134779876745,1.736030464994896,1.771674908313738,0.0030431481157541275,0.052864103123352636,5000.0,4.505511642675647,1900000 +0.26689332837612056,17.0,0.4898979485566356,0.04400441245696375,-0.025010323058401753,-213.70571513031055,0.014671253624856752,85.10985459487907,2.107897780535277,2.1560866539717183,0.0050599128153617475,0.06624935851298078,5000.0,4.515254162016503,1950000 +0.27264955870061425,47.45,24.374730767743877,-0.15417599499016782,0.2288789327870835,-217.2908005645753,-0.0055982787592335995,76.38042194450361,1.9874657223844558,2.086140009138611,0.0036129235415086145,0.0563990519023725,5000.0,4.516976481942977,2000000 +0.2650684246462617,22.03,19.580324307835145,0.12385188264204913,0.6911802229002474,-215.38197962005552,0.001599677744731283,69.3499005844138,1.9058774832868581,1.9457967880392086,0.002039753596700728,0.04212972788426176,5000.0,4.505184910560991,2050000 +0.2674392857334078,80.03,9.242786376412688,3.490060478763609,0.8491785106293334,-235.91733949951077,-0.003447431826256192,67.15164657203563,1.8901985336494316,1.954124572350993,0.0037300027757985513,0.05757505918758763,5000.0,4.505735197702261,2100000 +0.25453522945165874,38.82,34.566856958653325,0.6837963807038171,1.9927026384453321,-240.3333819339019,0.004707467174399603,66.1967412495418,1.917481872596723,1.9404282009470297,0.004822385240381581,0.06425157026775816,5000.0,4.502934927679571,2150000 +0.24501048744529552,19.0,6.549809157525126,0.026483052183666863,-0.02054705198329038,-244.01531899719023,-0.011420958120822979,60.956044562225095,1.8113102264285252,1.8833981944227085,0.0024422518733232274,0.04596810453107855,5000.0,4.501531263191133,2200000 +0.23920006792098158,49.74,11.066724899445182,-0.23462908571318725,0.04267156398758768,-249.5893155923453,-0.004355245703067407,65.79538648376538,1.8810041777062205,2.0200267978310507,0.0020851629534914397,0.043642319784713404,5000.0,4.502764927963597,2250000 +0.2289381713590033,260.08,234.21838868884743,10.022026163812004,16.395553885538025,-248.90724262298633,0.006540574790723724,64.32709826297838,1.8365286173939699,1.985385626833463,0.0022627882908023886,0.04477300858930074,5000.0,4.501002459680021,2300000 +0.21329243528515332,239.7,182.2494718785215,14.834931369918635,17.66472519841111,-252.4532543774414,-0.0016269144619252994,57.95849099578781,1.7394086742496495,1.8776908208239056,0.002652148633623729,0.04869599329505153,5000.0,4.496618398711671,2350000 +0.20907698753863627,154.36,12.425393353934513,7.2710357036836415,1.093844730572829,-250.79874050201445,-0.008132204290628386,56.06229531982365,1.738771745092882,1.8772951727366527,0.0020887383768223226,0.043271266660690914,5000.0,4.495087682535405,2400000 +0.2086045206478224,417.91,348.3750018299246,29.433457326263547,39.15040678406864,-245.68947620269537,0.006354975852239751,59.52232734104209,1.764057544300565,1.9553604222643362,0.0023110258430130773,0.04568485719588564,5000.0,4.502501755099615,2450000 +0.21026154054999144,584.89,396.3158814632591,55.186059387354355,42.35386280674607,-249.43521510254007,-0.012244713369608026,57.03046368301407,1.7450066427326298,1.8785939013051929,0.0022567787587176986,0.04527302006815237,5000.0,4.5012344624063125,2500000 +0.19474591458469706,847.07,316.9665677954064,88.58389880588722,36.54396219507402,-260.4634774624615,0.003303364705182582,53.90271531295754,1.7012549902343843,1.8382558945024075,0.0017169779576621952,0.039562200158826236,5000.0,4.499433216042866,2550000 +0.1972507061007596,649.44,372.70147088521134,11.668112948451688,8.533166383132158,-260.17022409577044,0.0006233002858237936,55.139105852317726,1.725274553458699,1.852137920080435,0.0025367046725985128,0.04755290351226989,5000.0,4.501970136785709,2600000 +0.20787622935831637,921.6,132.4976980932122,94.73494916240413,15.28163006732555,-266.55361816955286,0.007967453341484033,58.04983495697055,1.7775316660785472,1.9235283771991831,0.0031157156795645134,0.05397916215097938,5000.0,4.503641902745764,2650000 +0.20774135457217688,358.48,331.86293797289267,34.43084159301356,39.64812597518137,-263.0718863522316,-0.01425434614973136,58.59453063468848,1.7898780991935932,1.932939710309512,0.0027885479128845025,0.051022958052939285,5000.0,4.506539250334913,2700000 +0.20730049267709236,212.14,10.46233243593416,14.383009617787337,1.0716252601083625,-260.97839515106193,0.016841382037159118,65.88666886733967,1.895607242000129,2.106605510585318,0.003175444983644875,0.05453854898827052,5000.0,4.49159671298613,2750000 +0.19775769947111638,994.35,56.21679019652402,98.90002110489398,6.470278314124219,-256.7464621023534,0.003148048784993703,70.58843944610561,1.9635375590825144,2.171936205189202,0.0026679667041711506,0.049868863721067774,5000.0,4.4942559085119616,2800000 +0.19519495781183216,908.26,191.1622671972688,87.16084553366838,23.481998495017102,-248.08049752685733,0.009836789774522143,68.70194338169065,1.9466276859116636,2.1243337889957123,0.002823692617746069,0.05142471343127669,5000.0,4.491613867635841,2850000 +0.19303232329815376,193.34,2.490863304157817,10.058251904304333,0.16979519365976417,-248.35211029418898,-0.003225053575597882,71.22867902912083,1.9862170424413723,2.1668187445807305,0.0035707274847342632,0.05793158801028479,5000.0,4.492283627116287,2900000 +0.1928628791424622,171.29,83.8651649971548,9.71632145981134,9.246381082866662,-255.21302824585067,-0.014482733757831608,73.59544960170781,2.0727920477008612,2.2127810332083615,0.0036877333356607725,0.05905830305353895,5000.0,4.493799853047944,2950000 +0.2025139325544238,1000.0,0.0,104.7556684671196,0.9620606261445638,-235.48156189605828,0.04652526318948734,72.44967312034537,1.985553123996253,2.20009885561945,0.0037184259993275626,0.05936980585254698,5000.0,4.50001881703551,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193338/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193338/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..b5ee73d90848b7f2be743bae9006998b71ddf270 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193338/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 5000, reward mean: 4.4026 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5590 +num rollout transitions: 5000, reward mean: 4.5737 +num rollout transitions: 5000, reward mean: 4.5362 +num rollout transitions: 5000, reward mean: 4.4551 +num rollout transitions: 5000, reward mean: 4.4728 +num rollout transitions: 5000, reward mean: 4.4710 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.5392 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.5346 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.4851 +num rollout transitions: 5000, reward mean: 4.4177 +num rollout transitions: 5000, reward mean: 4.5771 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.5252 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.4681 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4749 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.4629 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.4607 +num rollout transitions: 5000, reward mean: 4.5400 +num rollout transitions: 5000, reward mean: 4.5350 +num rollout transitions: 5000, reward mean: 4.5471 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.4884 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.5439 +num rollout transitions: 5000, reward mean: 4.5548 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5170 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.5360 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.4839 +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.4936 +---------------------------------------------------------------------------------- +| alpha | 0.274 | +| eval/episode_length | 58.8 | +| eval/episode_length_std | 3.61 | +| eval/normalized_episode_reward | 0.0482 | +| eval/normalized_episode_reward_std | -0.0098 | +| loss/actor | -299 | +| loss/alpha | -1.74 | +| loss/critic | 25.2 | +| penalty/lcb_mean | 1.1 | +| penalty/lcb_std | 1.47 | +| rollout_info/dynamics_penalty_mean | 0.0165 | +| rollout_info/dynamics_penalty_std | 0.108 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.5509 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.4773 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5540 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.5288 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5278 +num rollout transitions: 5000, reward mean: 4.4988 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5258 +num rollout transitions: 5000, reward mean: 4.5425 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.5301 +num rollout transitions: 5000, reward mean: 4.5506 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.5397 +num rollout transitions: 5000, reward mean: 4.5352 +num rollout transitions: 5000, reward mean: 4.5426 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.5333 +num rollout transitions: 5000, reward mean: 4.5317 +num rollout transitions: 5000, reward mean: 4.4872 +num rollout transitions: 5000, reward mean: 4.5444 +num rollout transitions: 5000, reward mean: 4.5277 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.5438 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.4792 +num rollout transitions: 5000, reward mean: 4.4738 +num rollout transitions: 5000, reward mean: 4.4779 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5314 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.5266 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.5086 +---------------------------------------------------------------------------------- +| alpha | 0.157 | +| eval/episode_length | 97.5 | +| eval/episode_length_std | 23.4 | +| eval/normalized_episode_reward | 4.91 | +| eval/normalized_episode_reward_std | 2.23 | +| loss/actor | -421 | +| loss/alpha | -0.0441 | +| loss/critic | 32.7 | +| penalty/lcb_mean | 1.23 | +| penalty/lcb_std | 1.82 | +| rollout_info/dynamics_penalty_mean | 0.00265 | +| rollout_info/dynamics_penalty_std | 0.0479 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5601 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.5549 +num rollout transitions: 5000, reward mean: 4.4923 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.5154 +num rollout transitions: 5000, reward mean: 4.5604 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.5432 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.5298 +num rollout transitions: 5000, reward mean: 4.4681 +num rollout transitions: 5000, reward mean: 4.4646 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.5255 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.5197 +num rollout transitions: 5000, reward mean: 4.5312 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5376 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.5228 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.5072 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.5336 +num rollout transitions: 5000, reward mean: 4.5265 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.5638 +num rollout transitions: 5000, reward mean: 4.5296 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.4756 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.5284 +num rollout transitions: 5000, reward mean: 4.5456 +num rollout transitions: 5000, reward mean: 4.5582 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5454 +---------------------------------------------------------------------------------- +| alpha | 0.154 | +| eval/episode_length | 178 | +| eval/episode_length_std | 3.8 | +| eval/normalized_episode_reward | 10.5 | +| eval/normalized_episode_reward_std | 0.334 | +| loss/actor | -423 | +| loss/alpha | 0.00513 | +| loss/critic | 30.2 | +| penalty/lcb_mean | 1.22 | +| penalty/lcb_std | 1.86 | +| rollout_info/dynamics_penalty_mean | 0.00348 | +| rollout_info/dynamics_penalty_std | 0.0539 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5455 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5522 +num rollout transitions: 5000, reward mean: 4.5279 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.5324 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.4840 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.4762 +num rollout transitions: 5000, reward mean: 4.5539 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.4818 +num rollout transitions: 5000, reward mean: 4.5264 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5293 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.5427 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.5170 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.5565 +num rollout transitions: 5000, reward mean: 4.5369 +num rollout transitions: 5000, reward mean: 4.5171 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.5362 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.4852 +num rollout transitions: 5000, reward mean: 4.5279 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.4852 +num rollout transitions: 5000, reward mean: 4.5099 +---------------------------------------------------------------------------------- +| alpha | 0.169 | +| eval/episode_length | 124 | +| eval/episode_length_std | 10.1 | +| eval/normalized_episode_reward | 5.32 | +| eval/normalized_episode_reward_std | 0.914 | +| loss/actor | -422 | +| loss/alpha | 0.02 | +| loss/critic | 34.6 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 2.01 | +| rollout_info/dynamics_penalty_mean | 0.00341 | +| rollout_info/dynamics_penalty_std | 0.0545 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4769 +num rollout transitions: 5000, reward mean: 4.5279 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.4713 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.4672 +num rollout transitions: 5000, reward mean: 4.5363 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.5491 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.5255 +num rollout transitions: 5000, reward mean: 4.5222 +num rollout transitions: 5000, reward mean: 4.5356 +num rollout transitions: 5000, reward mean: 4.5454 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.5291 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5197 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.5574 +num rollout transitions: 5000, reward mean: 4.4879 +num rollout transitions: 5000, reward mean: 4.5659 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5370 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.4810 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5638 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.5420 +---------------------------------------------------------------------------------- +| alpha | 0.162 | +| eval/episode_length | 98.8 | +| eval/episode_length_std | 36.8 | +| eval/normalized_episode_reward | 5.28 | +| eval/normalized_episode_reward_std | 2.69 | +| loss/actor | -432 | +| loss/alpha | -0.0203 | +| loss/critic | 31.6 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 0.00286 | +| rollout_info/dynamics_penalty_std | 0.0499 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.4890 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.5013 +num rollout transitions: 5000, reward mean: 4.5500 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.5165 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.5469 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.5309 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.5469 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.5475 +num rollout transitions: 5000, reward mean: 4.5402 +num rollout transitions: 5000, reward mean: 4.4745 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.4955 +num rollout transitions: 5000, reward mean: 4.5379 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.4774 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.4927 +num rollout transitions: 5000, reward mean: 4.5370 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.5514 +num rollout transitions: 5000, reward mean: 4.5382 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.5216 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5228 +num rollout transitions: 5000, reward mean: 4.4773 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.5170 +num rollout transitions: 5000, reward mean: 4.5093 +---------------------------------------------------------------------------------- +| alpha | 0.159 | +| eval/episode_length | 121 | +| eval/episode_length_std | 9.6 | +| eval/normalized_episode_reward | 5.35 | +| eval/normalized_episode_reward_std | 0.44 | +| loss/actor | -428 | +| loss/alpha | 0.0141 | +| loss/critic | 31.6 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 1.9 | +| rollout_info/dynamics_penalty_mean | 0.00303 | +| rollout_info/dynamics_penalty_std | 0.0524 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.4819 +num rollout transitions: 5000, reward mean: 4.4923 +num rollout transitions: 5000, reward mean: 4.4762 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5388 +num rollout transitions: 5000, reward mean: 4.5080 +num rollout transitions: 5000, reward mean: 4.5312 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.5372 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.5455 +num rollout transitions: 5000, reward mean: 4.5591 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5557 +num rollout transitions: 5000, reward mean: 4.5180 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.5367 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.5332 +num rollout transitions: 5000, reward mean: 4.5318 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.5021 +num rollout transitions: 5000, reward mean: 4.5222 +num rollout transitions: 5000, reward mean: 4.5510 +num rollout transitions: 5000, reward mean: 4.5464 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.5636 +num rollout transitions: 5000, reward mean: 4.5174 +num rollout transitions: 5000, reward mean: 4.5302 +num rollout transitions: 5000, reward mean: 4.5242 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.5274 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.5017 +num rollout transitions: 5000, reward mean: 4.5385 +num rollout transitions: 5000, reward mean: 4.5351 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5274 +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.4982 +----------------------------------------------------------------------------------- +| alpha | 0.168 | +| eval/episode_length | 77.1 | +| eval/episode_length_std | 1.84 | +| eval/normalized_episode_reward | 4.05 | +| eval/normalized_episode_reward_std | 0.0934 | +| loss/actor | -405 | +| loss/alpha | -0.000715 | +| loss/critic | 35.5 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2 | +| rollout_info/dynamics_penalty_mean | 0.00268 | +| rollout_info/dynamics_penalty_std | 0.049 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.52 | +| timestep | 350000 | +----------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5436 +num rollout transitions: 5000, reward mean: 4.5162 +num rollout transitions: 5000, reward mean: 4.5312 +num rollout transitions: 5000, reward mean: 4.4801 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.5332 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.5259 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.5292 +num rollout transitions: 5000, reward mean: 4.5319 +num rollout transitions: 5000, reward mean: 4.5455 +num rollout transitions: 5000, reward mean: 4.5168 +num rollout transitions: 5000, reward mean: 4.5306 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5325 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.5324 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.4720 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.5312 +num rollout transitions: 5000, reward mean: 4.5320 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5203 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.5325 +num rollout transitions: 5000, reward mean: 4.5329 +num rollout transitions: 5000, reward mean: 4.5436 +num rollout transitions: 5000, reward mean: 4.5457 +num rollout transitions: 5000, reward mean: 4.5055 +---------------------------------------------------------------------------------- +| alpha | 0.168 | +| eval/episode_length | 118 | +| eval/episode_length_std | 59.7 | +| eval/normalized_episode_reward | 7.26 | +| eval/normalized_episode_reward_std | 6.25 | +| loss/actor | -396 | +| loss/alpha | 0.00889 | +| loss/critic | 35.6 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 1.93 | +| rollout_info/dynamics_penalty_mean | 0.00418 | +| rollout_info/dynamics_penalty_std | 0.0604 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.5430 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.5319 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.4863 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.4644 +num rollout transitions: 5000, reward mean: 4.5457 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4954 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.5279 +num rollout transitions: 5000, reward mean: 4.5269 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.5334 +num rollout transitions: 5000, reward mean: 4.5356 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5290 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4788 +num rollout transitions: 5000, reward mean: 4.4769 +num rollout transitions: 5000, reward mean: 4.4806 +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.5329 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.4822 +num rollout transitions: 5000, reward mean: 4.5506 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.4777 +num rollout transitions: 5000, reward mean: 4.5308 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5403 +---------------------------------------------------------------------------------- +| alpha | 0.164 | +| eval/episode_length | 183 | +| eval/episode_length_std | 13.8 | +| eval/normalized_episode_reward | 12.2 | +| eval/normalized_episode_reward_std | 1.41 | +| loss/actor | -392 | +| loss/alpha | -0.0102 | +| loss/critic | 36.1 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 1.97 | +| rollout_info/dynamics_penalty_mean | 0.00286 | +| rollout_info/dynamics_penalty_std | 0.0506 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5286 +num rollout transitions: 5000, reward mean: 4.4631 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.5418 +num rollout transitions: 5000, reward mean: 4.4741 +num rollout transitions: 5000, reward mean: 4.4884 +num rollout transitions: 5000, reward mean: 4.5478 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.4763 +num rollout transitions: 5000, reward mean: 4.4748 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.5317 +num rollout transitions: 5000, reward mean: 4.5217 +num rollout transitions: 5000, reward mean: 4.5421 +num rollout transitions: 5000, reward mean: 4.4738 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.5338 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.5386 +num rollout transitions: 5000, reward mean: 4.5238 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.4821 +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.4825 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5177 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.4944 +num rollout transitions: 5000, reward mean: 4.4802 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.5114 +---------------------------------------------------------------------------------- +| alpha | 0.168 | +| eval/episode_length | 345 | +| eval/episode_length_std | 94.7 | +| eval/normalized_episode_reward | 30.4 | +| eval/normalized_episode_reward_std | 11 | +| loss/actor | -388 | +| loss/alpha | -0.0041 | +| loss/critic | 40.8 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.18 | +| rollout_info/dynamics_penalty_mean | 0.00281 | +| rollout_info/dynamics_penalty_std | 0.0504 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.5399 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.5558 +num rollout transitions: 5000, reward mean: 4.5317 +num rollout transitions: 5000, reward mean: 4.5374 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.5192 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.5253 +num rollout transitions: 5000, reward mean: 4.5628 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.5365 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.5265 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.4902 +num rollout transitions: 5000, reward mean: 4.4757 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.4828 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5390 +num rollout transitions: 5000, reward mean: 4.4842 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.4617 +num rollout transitions: 5000, reward mean: 4.5563 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.5420 +num rollout transitions: 5000, reward mean: 4.5267 +---------------------------------------------------------------------------------- +| alpha | 0.173 | +| eval/episode_length | 996 | +| eval/episode_length_std | 20.5 | +| eval/normalized_episode_reward | 94.6 | +| eval/normalized_episode_reward_std | 1.99 | +| loss/actor | -385 | +| loss/alpha | 0.0209 | +| loss/critic | 43.3 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.18 | +| rollout_info/dynamics_penalty_mean | 0.00242 | +| rollout_info/dynamics_penalty_std | 0.0463 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.4817 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.5168 +num rollout transitions: 5000, reward mean: 4.5251 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5427 +num rollout transitions: 5000, reward mean: 4.5256 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.5478 +num rollout transitions: 5000, reward mean: 4.5554 +num rollout transitions: 5000, reward mean: 4.5329 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.5220 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5340 +num rollout transitions: 5000, reward mean: 4.5342 +num rollout transitions: 5000, reward mean: 4.5128 +num rollout transitions: 5000, reward mean: 4.5249 +num rollout transitions: 5000, reward mean: 4.5448 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.5193 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.5373 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5507 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5292 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.4778 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.5241 +num rollout transitions: 5000, reward mean: 4.5016 +---------------------------------------------------------------------------------- +| alpha | 0.178 | +| eval/episode_length | 438 | +| eval/episode_length_std | 370 | +| eval/normalized_episode_reward | 39.8 | +| eval/normalized_episode_reward_std | 40 | +| loss/actor | -372 | +| loss/alpha | -0.00836 | +| loss/critic | 50.8 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 0.00241 | +| rollout_info/dynamics_penalty_std | 0.047 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4902 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.5115 +num rollout transitions: 5000, reward mean: 4.5413 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.4812 +num rollout transitions: 5000, reward mean: 4.4624 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4681 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.5217 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.5240 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.5371 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.4736 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.4662 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5263 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.5319 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5449 +num rollout transitions: 5000, reward mean: 4.5387 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5333 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.4928 +---------------------------------------------------------------------------------- +| alpha | 0.174 | +| eval/episode_length | 12 | +| eval/episode_length_std | 0.14 | +| eval/normalized_episode_reward | -0.255 | +| eval/normalized_episode_reward_std | -0.0288 | +| loss/actor | -361 | +| loss/alpha | -0.00242 | +| loss/critic | 51.4 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 0.00301 | +| rollout_info/dynamics_penalty_std | 0.052 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.5071 +num rollout transitions: 5000, reward mean: 4.5477 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.5441 +num rollout transitions: 5000, reward mean: 4.5536 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.5013 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5271 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.4904 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.5265 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.5348 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.5265 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.5673 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5349 +num rollout transitions: 5000, reward mean: 4.5314 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.5316 +---------------------------------------------------------------------------------- +| alpha | 0.197 | +| eval/episode_length | 90.2 | +| eval/episode_length_std | 11.6 | +| eval/normalized_episode_reward | 3.67 | +| eval/normalized_episode_reward_std | 0.765 | +| loss/actor | -360 | +| loss/alpha | -0.00838 | +| loss/critic | 51.1 | +| penalty/lcb_mean | 1.56 | +| penalty/lcb_std | 2.29 | +| rollout_info/dynamics_penalty_mean | 0.00275 | +| rollout_info/dynamics_penalty_std | 0.0491 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.5588 +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5479 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.5323 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.4884 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.5241 +num rollout transitions: 5000, reward mean: 4.4822 +num rollout transitions: 5000, reward mean: 4.5461 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5274 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.4841 +num rollout transitions: 5000, reward mean: 4.4788 +num rollout transitions: 5000, reward mean: 4.5238 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5493 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.4668 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.4949 +---------------------------------------------------------------------------------- +| alpha | 0.192 | +| eval/episode_length | 810 | +| eval/episode_length_std | 151 | +| eval/normalized_episode_reward | 84.1 | +| eval/normalized_episode_reward_std | 18.1 | +| loss/actor | -354 | +| loss/alpha | 0.0242 | +| loss/critic | 50.6 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.29 | +| rollout_info/dynamics_penalty_mean | 0.00221 | +| rollout_info/dynamics_penalty_std | 0.044 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.5281 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.4927 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.4775 +num rollout transitions: 5000, reward mean: 4.5269 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.4684 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.5165 +num rollout transitions: 5000, reward mean: 4.5283 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.4949 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5288 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.4748 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.4949 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5286 +num rollout transitions: 5000, reward mean: 4.5404 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.4917 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.5248 +num rollout transitions: 5000, reward mean: 4.4955 +num rollout transitions: 5000, reward mean: 4.4803 +num rollout transitions: 5000, reward mean: 4.5276 +num rollout transitions: 5000, reward mean: 4.4885 +num rollout transitions: 5000, reward mean: 4.4744 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.5157 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.4845 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.4695 +num rollout transitions: 5000, reward mean: 4.4927 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.4796 +num rollout transitions: 5000, reward mean: 4.4752 +---------------------------------------------------------------------------------- +| alpha | 0.185 | +| eval/episode_length | 101 | +| eval/episode_length_std | 3.75 | +| eval/normalized_episode_reward | 4.92 | +| eval/normalized_episode_reward_std | 0.245 | +| loss/actor | -353 | +| loss/alpha | -0.0367 | +| loss/critic | 49 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.3 | +| rollout_info/dynamics_penalty_mean | 0.002 | +| rollout_info/dynamics_penalty_std | 0.0415 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4720 +num rollout transitions: 5000, reward mean: 4.4882 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.5374 +num rollout transitions: 5000, reward mean: 4.5334 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.5278 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4845 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4840 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5222 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.5248 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.5301 +num rollout transitions: 5000, reward mean: 4.4736 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.4663 +num rollout transitions: 5000, reward mean: 4.4878 +num rollout transitions: 5000, reward mean: 4.5280 +num rollout transitions: 5000, reward mean: 4.5348 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.4864 +---------------------------------------------------------------------------------- +| alpha | 0.186 | +| eval/episode_length | 922 | +| eval/episode_length_std | 218 | +| eval/normalized_episode_reward | 88.5 | +| eval/normalized_episode_reward_std | 22.6 | +| loss/actor | -359 | +| loss/alpha | 0.0316 | +| loss/critic | 51.6 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0.00218 | +| rollout_info/dynamics_penalty_std | 0.0442 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4955 +num rollout transitions: 5000, reward mean: 4.4896 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.5019 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.4728 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.4666 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.4828 +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5338 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.5336 +num rollout transitions: 5000, reward mean: 4.5407 +num rollout transitions: 5000, reward mean: 4.4963 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.5575 +num rollout transitions: 5000, reward mean: 4.4804 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.4944 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.4810 +num rollout transitions: 5000, reward mean: 4.5194 +num rollout transitions: 5000, reward mean: 4.5415 +num rollout transitions: 5000, reward mean: 4.4800 +num rollout transitions: 5000, reward mean: 4.4863 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5262 +---------------------------------------------------------------------------------- +| alpha | 0.193 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 98.9 | +| eval/normalized_episode_reward_std | 0.872 | +| loss/actor | -356 | +| loss/alpha | 0.00342 | +| loss/critic | 55.6 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.44 | +| rollout_info/dynamics_penalty_mean | 0.00176 | +| rollout_info/dynamics_penalty_std | 0.0395 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.5238 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.4789 +num rollout transitions: 5000, reward mean: 4.5317 +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.4483 +num rollout transitions: 5000, reward mean: 4.4651 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.4700 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.4652 +num rollout transitions: 5000, reward mean: 4.5297 +num rollout transitions: 5000, reward mean: 4.4703 +num rollout transitions: 5000, reward mean: 4.4580 +num rollout transitions: 5000, reward mean: 4.4648 +num rollout transitions: 5000, reward mean: 4.4715 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.4841 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.4403 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.4753 +num rollout transitions: 5000, reward mean: 4.5308 +---------------------------------------------------------------------------------- +| alpha | 0.204 | +| eval/episode_length | 998 | +| eval/episode_length_std | 22.4 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 2.79 | +| loss/actor | -339 | +| loss/alpha | -0.0013 | +| loss/critic | 59.6 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.47 | +| rollout_info/dynamics_penalty_mean | 0.00163 | +| rollout_info/dynamics_penalty_std | 0.0378 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.5065 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.5157 +num rollout transitions: 5000, reward mean: 4.4746 +num rollout transitions: 5000, reward mean: 4.4638 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5473 +num rollout transitions: 5000, reward mean: 4.5609 +num rollout transitions: 5000, reward mean: 4.4806 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.4624 +num rollout transitions: 5000, reward mean: 4.5306 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.5409 +num rollout transitions: 5000, reward mean: 4.5115 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.5353 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.4619 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.4838 +num rollout transitions: 5000, reward mean: 4.4842 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5275 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.4944 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.5219 +---------------------------------------------------------------------------------- +| alpha | 0.192 | +| eval/episode_length | 179 | +| eval/episode_length_std | 6.49 | +| eval/normalized_episode_reward | 11.3 | +| eval/normalized_episode_reward_std | 0.64 | +| loss/actor | -340 | +| loss/alpha | -0.0144 | +| loss/critic | 61.3 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.52 | +| rollout_info/dynamics_penalty_mean | 0.00171 | +| rollout_info/dynamics_penalty_std | 0.0391 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.4752 +num rollout transitions: 5000, reward mean: 4.5344 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.4761 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.4757 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.5242 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.4952 +---------------------------------------------------------------------------------- +| alpha | 0.203 | +| eval/episode_length | 168 | +| eval/episode_length_std | 84.4 | +| eval/normalized_episode_reward | 10.2 | +| eval/normalized_episode_reward_std | 8.23 | +| loss/actor | -335 | +| loss/alpha | 0.00586 | +| loss/critic | 60.7 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.52 | +| rollout_info/dynamics_penalty_mean | 0.00196 | +| rollout_info/dynamics_penalty_std | 0.0417 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5393 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.5168 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.5351 +num rollout transitions: 5000, reward mean: 4.5379 +num rollout transitions: 5000, reward mean: 4.5305 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4832 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.5526 +num rollout transitions: 5000, reward mean: 4.5347 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.4930 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.4861 +num rollout transitions: 5000, reward mean: 4.5381 +num rollout transitions: 5000, reward mean: 4.4878 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.4804 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.4801 +num rollout transitions: 5000, reward mean: 4.4884 +num rollout transitions: 5000, reward mean: 4.4758 +num rollout transitions: 5000, reward mean: 4.4890 +num rollout transitions: 5000, reward mean: 4.5221 +num rollout transitions: 5000, reward mean: 4.4826 +num rollout transitions: 5000, reward mean: 4.4842 +num rollout transitions: 5000, reward mean: 4.5417 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.5345 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.4940 +num rollout transitions: 5000, reward mean: 4.4777 +num rollout transitions: 5000, reward mean: 4.4736 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4829 +num rollout transitions: 5000, reward mean: 4.4713 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5028 +---------------------------------------------------------------------------------- +| alpha | 0.207 | +| eval/episode_length | 875 | +| eval/episode_length_std | 195 | +| eval/normalized_episode_reward | 91.8 | +| eval/normalized_episode_reward_std | 22.1 | +| loss/actor | -338 | +| loss/alpha | 0.00696 | +| loss/critic | 60.2 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0.00145 | +| rollout_info/dynamics_penalty_std | 0.0356 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.4624 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.4747 +num rollout transitions: 5000, reward mean: 4.4955 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.4759 +num rollout transitions: 5000, reward mean: 4.4791 +num rollout transitions: 5000, reward mean: 4.5310 +num rollout transitions: 5000, reward mean: 4.5013 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.4530 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.5376 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.5315 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.4729 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.4819 +num rollout transitions: 5000, reward mean: 4.4818 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.4850 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.4801 +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.4782 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.5162 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.4878 +num rollout transitions: 5000, reward mean: 4.4923 +num rollout transitions: 5000, reward mean: 4.4636 +---------------------------------------------------------------------------------- +| alpha | 0.202 | +| eval/episode_length | 934 | +| eval/episode_length_std | 152 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 17.7 | +| loss/actor | -339 | +| loss/alpha | -0.00733 | +| loss/critic | 66.9 | +| penalty/lcb_mean | 1.73 | +| penalty/lcb_std | 2.64 | +| rollout_info/dynamics_penalty_mean | 0.00171 | +| rollout_info/dynamics_penalty_std | 0.0393 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5184 +num rollout transitions: 5000, reward mean: 4.4823 +num rollout transitions: 5000, reward mean: 4.5017 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.4883 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4659 +num rollout transitions: 5000, reward mean: 4.4701 +num rollout transitions: 5000, reward mean: 4.4868 +num rollout transitions: 5000, reward mean: 4.4761 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5237 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.4678 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.4653 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.4818 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.4818 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.4698 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.4588 +num rollout transitions: 5000, reward mean: 4.4788 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4707 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.4678 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.4791 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4845 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.4720 +num rollout transitions: 5000, reward mean: 4.4591 +---------------------------------------------------------------------------------- +| alpha | 0.208 | +| eval/episode_length | 198 | +| eval/episode_length_std | 8.86 | +| eval/normalized_episode_reward | 14.2 | +| eval/normalized_episode_reward_std | 0.818 | +| loss/actor | -329 | +| loss/alpha | 0.00653 | +| loss/critic | 63.6 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.47 | +| rollout_info/dynamics_penalty_mean | 0.00189 | +| rollout_info/dynamics_penalty_std | 0.041 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.49 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4631 +num rollout transitions: 5000, reward mean: 4.4561 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.4679 +num rollout transitions: 5000, reward mean: 4.4963 +num rollout transitions: 5000, reward mean: 4.4698 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.4824 +num rollout transitions: 5000, reward mean: 4.4452 +num rollout transitions: 5000, reward mean: 4.4949 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.4869 +num rollout transitions: 5000, reward mean: 4.4940 +num rollout transitions: 5000, reward mean: 4.5154 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.4766 +num rollout transitions: 5000, reward mean: 4.4821 +num rollout transitions: 5000, reward mean: 4.4759 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.5476 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4740 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4892 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.4821 +---------------------------------------------------------------------------------- +| alpha | 0.211 | +| eval/episode_length | 267 | +| eval/episode_length_std | 183 | +| eval/normalized_episode_reward | 23.2 | +| eval/normalized_episode_reward_std | 20.7 | +| loss/actor | -338 | +| loss/alpha | 0.00174 | +| loss/critic | 68.4 | +| penalty/lcb_mean | 1.79 | +| penalty/lcb_std | 2.64 | +| rollout_info/dynamics_penalty_mean | 0.00162 | +| rollout_info/dynamics_penalty_std | 0.0382 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.4795 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.5453 +num rollout transitions: 5000, reward mean: 4.4764 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.5171 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.4731 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.4727 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.4995 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4308 +num rollout transitions: 5000, reward mean: 4.4692 +num rollout transitions: 5000, reward mean: 4.4752 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4648 +num rollout transitions: 5000, reward mean: 4.4962 +num rollout transitions: 5000, reward mean: 4.4674 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.5298 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.4608 +num rollout transitions: 5000, reward mean: 4.4588 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.5072 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.5222 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.5361 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.4949 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.4799 +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.4894 +---------------------------------------------------------------------------------- +| alpha | 0.219 | +| eval/episode_length | 931 | +| eval/episode_length_std | 161 | +| eval/normalized_episode_reward | 96.7 | +| eval/normalized_episode_reward_std | 18.1 | +| loss/actor | -339 | +| loss/alpha | 0.00368 | +| loss/critic | 67.9 | +| penalty/lcb_mean | 1.8 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0.00128 | +| rollout_info/dynamics_penalty_std | 0.033 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5410 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.4764 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.4803 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.4690 +num rollout transitions: 5000, reward mean: 4.4829 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.5551 +num rollout transitions: 5000, reward mean: 4.5429 +num rollout transitions: 5000, reward mean: 4.5171 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.4766 +num rollout transitions: 5000, reward mean: 4.4627 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5397 +num rollout transitions: 5000, reward mean: 4.4817 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.5110 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.4710 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5065 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5188 +num rollout transitions: 5000, reward mean: 4.5480 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.4714 +num rollout transitions: 5000, reward mean: 4.5037 +---------------------------------------------------------------------------------- +| alpha | 0.216 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 0.922 | +| loss/actor | -336 | +| loss/alpha | -0.0117 | +| loss/critic | 71.5 | +| penalty/lcb_mean | 1.78 | +| penalty/lcb_std | 2.7 | +| rollout_info/dynamics_penalty_mean | 0.00138 | +| rollout_info/dynamics_penalty_std | 0.0352 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.5065 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.4791 +num rollout transitions: 5000, reward mean: 4.4754 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.4812 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.4838 +num rollout transitions: 5000, reward mean: 4.4833 +num rollout transitions: 5000, reward mean: 4.4814 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.4888 +num rollout transitions: 5000, reward mean: 4.4777 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.4854 +num rollout transitions: 5000, reward mean: 4.5213 +num rollout transitions: 5000, reward mean: 4.4716 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.4868 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.5289 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5369 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.4841 +---------------------------------------------------------------------------------- +| alpha | 0.223 | +| eval/episode_length | 182 | +| eval/episode_length_std | 84.2 | +| eval/normalized_episode_reward | 2.21 | +| eval/normalized_episode_reward_std | 8.73 | +| loss/actor | -338 | +| loss/alpha | 0.0109 | +| loss/critic | 70.7 | +| penalty/lcb_mean | 1.77 | +| penalty/lcb_std | 2.64 | +| rollout_info/dynamics_penalty_mean | 0.0016 | +| rollout_info/dynamics_penalty_std | 0.0374 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.4737 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.5061 +num rollout transitions: 5000, reward mean: 4.4842 +num rollout transitions: 5000, reward mean: 4.5266 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.4785 +num rollout transitions: 5000, reward mean: 4.4657 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.4678 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.4871 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.4769 +num rollout transitions: 5000, reward mean: 4.4765 +num rollout transitions: 5000, reward mean: 4.5297 +num rollout transitions: 5000, reward mean: 4.5325 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.4828 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.5071 +num rollout transitions: 5000, reward mean: 4.5311 +num rollout transitions: 5000, reward mean: 4.5076 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5434 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.4693 +---------------------------------------------------------------------------------- +| alpha | 0.217 | +| eval/episode_length | 586 | +| eval/episode_length_std | 406 | +| eval/normalized_episode_reward | 55 | +| eval/normalized_episode_reward_std | 44.3 | +| loss/actor | -338 | +| loss/alpha | -0.00373 | +| loss/critic | 65.8 | +| penalty/lcb_mean | 1.76 | +| penalty/lcb_std | 2.58 | +| rollout_info/dynamics_penalty_mean | 0.00158 | +| rollout_info/dynamics_penalty_std | 0.0375 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.5057 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.5327 +num rollout transitions: 5000, reward mean: 4.5395 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5310 +num rollout transitions: 5000, reward mean: 4.4822 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5295 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.5454 +num rollout transitions: 5000, reward mean: 4.5253 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.4742 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.4852 +num rollout transitions: 5000, reward mean: 4.5266 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.5548 +num rollout transitions: 5000, reward mean: 4.5296 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5247 +num rollout transitions: 5000, reward mean: 4.5288 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.5383 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.5446 +num rollout transitions: 5000, reward mean: 4.5222 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.5409 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.5234 +num rollout transitions: 5000, reward mean: 4.5374 +num rollout transitions: 5000, reward mean: 4.4963 +num rollout transitions: 5000, reward mean: 4.4963 +num rollout transitions: 5000, reward mean: 4.4874 +---------------------------------------------------------------------------------- +| alpha | 0.224 | +| eval/episode_length | 181 | +| eval/episode_length_std | 169 | +| eval/normalized_episode_reward | 13.4 | +| eval/normalized_episode_reward_std | 19.4 | +| loss/actor | -345 | +| loss/alpha | -0.00882 | +| loss/critic | 68.8 | +| penalty/lcb_mean | 1.77 | +| penalty/lcb_std | 2.71 | +| rollout_info/dynamics_penalty_mean | 0.00178 | +| rollout_info/dynamics_penalty_std | 0.0397 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.4623 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.5274 +num rollout transitions: 5000, reward mean: 4.5088 +num rollout transitions: 5000, reward mean: 4.4902 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5283 +num rollout transitions: 5000, reward mean: 4.5235 +num rollout transitions: 5000, reward mean: 4.5276 +num rollout transitions: 5000, reward mean: 4.5337 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.5271 +num rollout transitions: 5000, reward mean: 4.4448 +num rollout transitions: 5000, reward mean: 4.4710 +num rollout transitions: 5000, reward mean: 4.4900 +num rollout transitions: 5000, reward mean: 4.4732 +num rollout transitions: 5000, reward mean: 4.5448 +num rollout transitions: 5000, reward mean: 4.5379 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.4628 +num rollout transitions: 5000, reward mean: 4.5271 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.5375 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.5106 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.4943 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.4795 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.5070 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.5225 +num rollout transitions: 5000, reward mean: 4.4807 +---------------------------------------------------------------------------------- +| alpha | 0.212 | +| eval/episode_length | 938 | +| eval/episode_length_std | 161 | +| eval/normalized_episode_reward | 90.3 | +| eval/normalized_episode_reward_std | 18.3 | +| loss/actor | -337 | +| loss/alpha | 0.0178 | +| loss/critic | 71 | +| penalty/lcb_mean | 1.81 | +| penalty/lcb_std | 2.71 | +| rollout_info/dynamics_penalty_mean | 0.00206 | +| rollout_info/dynamics_penalty_std | 0.0428 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.5502 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.5197 +num rollout transitions: 5000, reward mean: 4.5249 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.4571 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.4827 +num rollout transitions: 5000, reward mean: 4.4769 +num rollout transitions: 5000, reward mean: 4.5248 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.5368 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.5335 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.5423 +num rollout transitions: 5000, reward mean: 4.4758 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.4821 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.5419 +num rollout transitions: 5000, reward mean: 4.5047 +num rollout transitions: 5000, reward mean: 4.4971 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5293 +num rollout transitions: 5000, reward mean: 4.4817 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.4806 +num rollout transitions: 5000, reward mean: 4.4774 +num rollout transitions: 5000, reward mean: 4.4785 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.5242 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5307 +num rollout transitions: 5000, reward mean: 4.5304 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.5203 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.4988 +---------------------------------------------------------------------------------- +| alpha | 0.218 | +| eval/episode_length | 527 | +| eval/episode_length_std | 295 | +| eval/normalized_episode_reward | 49.4 | +| eval/normalized_episode_reward_std | 33.5 | +| loss/actor | -334 | +| loss/alpha | -0.00758 | +| loss/critic | 72.9 | +| penalty/lcb_mean | 1.85 | +| penalty/lcb_std | 2.75 | +| rollout_info/dynamics_penalty_mean | 0.00241 | +| rollout_info/dynamics_penalty_std | 0.0463 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4787 +num rollout transitions: 5000, reward mean: 4.4569 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.4794 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.5262 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4773 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.5321 +num rollout transitions: 5000, reward mean: 4.4824 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.4792 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.5196 +num rollout transitions: 5000, reward mean: 4.5309 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.4896 +num rollout transitions: 5000, reward mean: 4.4644 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.4780 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5171 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.4595 +num rollout transitions: 5000, reward mean: 4.5429 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.4691 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.4722 +num rollout transitions: 5000, reward mean: 4.5240 +num rollout transitions: 5000, reward mean: 4.4680 +num rollout transitions: 5000, reward mean: 4.4885 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.5582 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4963 +---------------------------------------------------------------------------------- +| alpha | 0.218 | +| eval/episode_length | 363 | +| eval/episode_length_std | 288 | +| eval/normalized_episode_reward | 31.8 | +| eval/normalized_episode_reward_std | 33.1 | +| loss/actor | -335 | +| loss/alpha | -0.0104 | +| loss/critic | 81.8 | +| penalty/lcb_mean | 2.03 | +| penalty/lcb_std | 2.83 | +| rollout_info/dynamics_penalty_mean | 0.00169 | +| rollout_info/dynamics_penalty_std | 0.0386 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5174 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.4807 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.4899 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.4829 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.4753 +num rollout transitions: 5000, reward mean: 4.4863 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.4879 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.4833 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.5052 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5396 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4571 +num rollout transitions: 5000, reward mean: 4.5280 +num rollout transitions: 5000, reward mean: 4.4815 +num rollout transitions: 5000, reward mean: 4.5238 +num rollout transitions: 5000, reward mean: 4.5537 +num rollout transitions: 5000, reward mean: 4.5637 +num rollout transitions: 5000, reward mean: 4.5330 +num rollout transitions: 5000, reward mean: 4.5283 +num rollout transitions: 5000, reward mean: 4.5203 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.5213 +num rollout transitions: 5000, reward mean: 4.5474 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.5463 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.5355 +num rollout transitions: 5000, reward mean: 4.5386 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.5450 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5315 +num rollout transitions: 5000, reward mean: 4.5255 +num rollout transitions: 5000, reward mean: 4.5066 +---------------------------------------------------------------------------------- +| alpha | 0.22 | +| eval/episode_length | 995 | +| eval/episode_length_std | 40.4 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 4.93 | +| loss/actor | -342 | +| loss/alpha | 0.0133 | +| loss/critic | 83.5 | +| penalty/lcb_mean | 2.04 | +| penalty/lcb_std | 2.92 | +| rollout_info/dynamics_penalty_mean | 0.00259 | +| rollout_info/dynamics_penalty_std | 0.0489 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.4707 +num rollout transitions: 5000, reward mean: 4.4685 +num rollout transitions: 5000, reward mean: 4.4753 +num rollout transitions: 5000, reward mean: 4.5193 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.4846 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.5297 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.4663 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.5338 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.5261 +num rollout transitions: 5000, reward mean: 4.4693 +num rollout transitions: 5000, reward mean: 4.5178 +num rollout transitions: 5000, reward mean: 4.4789 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.5036 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.4834 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.5080 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.5128 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5319 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.5159 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.5325 +---------------------------------------------------------------------------------- +| alpha | 0.238 | +| eval/episode_length | 14 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | -0.155 | +| eval/normalized_episode_reward_std | -0.0333 | +| loss/actor | -325 | +| loss/alpha | 0.0123 | +| loss/critic | 81.5 | +| penalty/lcb_mean | 1.99 | +| penalty/lcb_std | 2.77 | +| rollout_info/dynamics_penalty_mean | 0.00215 | +| rollout_info/dynamics_penalty_std | 0.0439 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.5093 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.5225 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.5021 +num rollout transitions: 5000, reward mean: 4.5247 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4865 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4821 +num rollout transitions: 5000, reward mean: 4.4449 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.4589 +num rollout transitions: 5000, reward mean: 4.4704 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.5115 +num rollout transitions: 5000, reward mean: 4.4771 +num rollout transitions: 5000, reward mean: 4.5377 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.4980 +num rollout transitions: 5000, reward mean: 4.4607 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.4841 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.5095 +num rollout transitions: 5000, reward mean: 4.4841 +num rollout transitions: 5000, reward mean: 4.4806 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.4845 +num rollout transitions: 5000, reward mean: 4.4841 +num rollout transitions: 5000, reward mean: 4.4734 +num rollout transitions: 5000, reward mean: 4.5203 +---------------------------------------------------------------------------------- +| alpha | 0.236 | +| eval/episode_length | 819 | +| eval/episode_length_std | 328 | +| eval/normalized_episode_reward | 78.9 | +| eval/normalized_episode_reward_std | 34.2 | +| loss/actor | -323 | +| loss/alpha | -0.00914 | +| loss/critic | 81.1 | +| penalty/lcb_mean | 2.05 | +| penalty/lcb_std | 2.8 | +| rollout_info/dynamics_penalty_mean | 0.0024 | +| rollout_info/dynamics_penalty_std | 0.0463 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4511 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.5203 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.5527 +num rollout transitions: 5000, reward mean: 4.5473 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.4758 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.5072 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.4799 +num rollout transitions: 5000, reward mean: 4.4728 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.4831 +num rollout transitions: 5000, reward mean: 4.5177 +num rollout transitions: 5000, reward mean: 4.5510 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.5452 +num rollout transitions: 5000, reward mean: 4.4787 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.5397 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.5402 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5009 +num rollout transitions: 5000, reward mean: 4.5370 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.4585 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.5275 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5216 +num rollout transitions: 5000, reward mean: 4.5242 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5053 +---------------------------------------------------------------------------------- +| alpha | 0.211 | +| eval/episode_length | 925 | +| eval/episode_length_std | 206 | +| eval/normalized_episode_reward | 99.3 | +| eval/normalized_episode_reward_std | 24.3 | +| loss/actor | -328 | +| loss/alpha | -0.00421 | +| loss/critic | 76.3 | +| penalty/lcb_mean | 1.93 | +| penalty/lcb_std | 2.8 | +| rollout_info/dynamics_penalty_mean | 0.00212 | +| rollout_info/dynamics_penalty_std | 0.043 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4880 +num rollout transitions: 5000, reward mean: 4.5192 +num rollout transitions: 5000, reward mean: 4.4969 +num rollout transitions: 5000, reward mean: 4.5250 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5297 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.5559 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.5284 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5353 +num rollout transitions: 5000, reward mean: 4.5318 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.5237 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5308 +num rollout transitions: 5000, reward mean: 4.5188 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.4707 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.4718 +num rollout transitions: 5000, reward mean: 4.5458 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.4789 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5297 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.4882 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.4779 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.4944 +num rollout transitions: 5000, reward mean: 4.5595 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.4818 +---------------------------------------------------------------------------------- +| alpha | 0.224 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 0.981 | +| loss/actor | -328 | +| loss/alpha | 0.00652 | +| loss/critic | 80.3 | +| penalty/lcb_mean | 2.03 | +| penalty/lcb_std | 2.77 | +| rollout_info/dynamics_penalty_mean | 0.00188 | +| rollout_info/dynamics_penalty_std | 0.0411 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5310 +num rollout transitions: 5000, reward mean: 4.5383 +num rollout transitions: 5000, reward mean: 4.4939 +num rollout transitions: 5000, reward mean: 4.5328 +num rollout transitions: 5000, reward mean: 4.5288 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.5246 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4838 +num rollout transitions: 5000, reward mean: 4.5269 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.5300 +num rollout transitions: 5000, reward mean: 4.5164 +num rollout transitions: 5000, reward mean: 4.5448 +num rollout transitions: 5000, reward mean: 4.5312 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.5388 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.5227 +num rollout transitions: 5000, reward mean: 4.5331 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.5395 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.4704 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.4827 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5340 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5057 +---------------------------------------------------------------------------------- +| alpha | 0.225 | +| eval/episode_length | 683 | +| eval/episode_length_std | 366 | +| eval/normalized_episode_reward | 70.6 | +| eval/normalized_episode_reward_std | 43.4 | +| loss/actor | -325 | +| loss/alpha | -0.00765 | +| loss/critic | 83.2 | +| penalty/lcb_mean | 2.04 | +| penalty/lcb_std | 2.84 | +| rollout_info/dynamics_penalty_mean | 0.00194 | +| rollout_info/dynamics_penalty_std | 0.0419 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.5354 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.4910 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.4789 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.4976 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4994 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4782 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.5404 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.4884 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.4710 +num rollout transitions: 5000, reward mean: 4.4767 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.4855 +num rollout transitions: 5000, reward mean: 4.5383 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.5118 +num rollout transitions: 5000, reward mean: 4.5169 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.5256 +num rollout transitions: 5000, reward mean: 4.5160 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5011 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.4960 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.5201 +---------------------------------------------------------------------------------- +| alpha | 0.242 | +| eval/episode_length | 151 | +| eval/episode_length_std | 8.51 | +| eval/normalized_episode_reward | 10.5 | +| eval/normalized_episode_reward_std | 0.849 | +| loss/actor | -327 | +| loss/alpha | 0.00841 | +| loss/critic | 88 | +| penalty/lcb_mean | 2.16 | +| penalty/lcb_std | 2.88 | +| rollout_info/dynamics_penalty_mean | 0.00206 | +| rollout_info/dynamics_penalty_std | 0.0432 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.5267 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5310 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.5266 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.5106 +num rollout transitions: 5000, reward mean: 4.4645 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.4762 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.5302 +num rollout transitions: 5000, reward mean: 4.4710 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.4756 +num rollout transitions: 5000, reward mean: 4.5317 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.5227 +num rollout transitions: 5000, reward mean: 4.4858 +num rollout transitions: 5000, reward mean: 4.5139 +num rollout transitions: 5000, reward mean: 4.4893 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5583 +num rollout transitions: 5000, reward mean: 4.5336 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5245 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4761 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.4785 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.4544 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.4785 +num rollout transitions: 5000, reward mean: 4.4800 +---------------------------------------------------------------------------------- +| alpha | 0.251 | +| eval/episode_length | 163 | +| eval/episode_length_std | 14.9 | +| eval/normalized_episode_reward | 9.96 | +| eval/normalized_episode_reward_std | 1.05 | +| loss/actor | -342 | +| loss/alpha | 0.0102 | +| loss/critic | 98.6 | +| penalty/lcb_mean | 2.29 | +| penalty/lcb_std | 3.08 | +| rollout_info/dynamics_penalty_mean | 0.00253 | +| rollout_info/dynamics_penalty_std | 0.047 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4652 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.4858 +num rollout transitions: 5000, reward mean: 4.4818 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.5130 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.5172 +num rollout transitions: 5000, reward mean: 4.5626 +num rollout transitions: 5000, reward mean: 4.5311 +num rollout transitions: 5000, reward mean: 4.5352 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.5177 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.5004 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.5136 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.4949 +num rollout transitions: 5000, reward mean: 4.5241 +num rollout transitions: 5000, reward mean: 4.5275 +num rollout transitions: 5000, reward mean: 4.5328 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.4844 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.5087 +num rollout transitions: 5000, reward mean: 4.5545 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.4832 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5420 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.5294 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.4958 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.5360 +num rollout transitions: 5000, reward mean: 4.5432 +num rollout transitions: 5000, reward mean: 4.5324 +num rollout transitions: 5000, reward mean: 4.5307 +num rollout transitions: 5000, reward mean: 4.5291 +---------------------------------------------------------------------------------- +| alpha | 0.244 | +| eval/episode_length | 795 | +| eval/episode_length_std | 187 | +| eval/normalized_episode_reward | 90.9 | +| eval/normalized_episode_reward_std | 23.7 | +| loss/actor | -346 | +| loss/alpha | -0.0215 | +| loss/critic | 82.1 | +| penalty/lcb_mean | 2.11 | +| penalty/lcb_std | 2.76 | +| rollout_info/dynamics_penalty_mean | 0.00192 | +| rollout_info/dynamics_penalty_std | 0.0411 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.4717 +num rollout transitions: 5000, reward mean: 4.5278 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.4878 +num rollout transitions: 5000, reward mean: 4.5213 +num rollout transitions: 5000, reward mean: 4.5541 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.5222 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.5459 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.5516 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.4953 +num rollout transitions: 5000, reward mean: 4.5204 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.5353 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.5096 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5133 +num rollout transitions: 5000, reward mean: 4.4902 +num rollout transitions: 5000, reward mean: 4.5247 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.5354 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.4837 +num rollout transitions: 5000, reward mean: 4.4821 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4727 +num rollout transitions: 5000, reward mean: 4.4867 +num rollout transitions: 5000, reward mean: 4.5346 +num rollout transitions: 5000, reward mean: 4.5400 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.5205 +---------------------------------------------------------------------------------- +| alpha | 0.231 | +| eval/episode_length | 910 | +| eval/episode_length_std | 213 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 26.6 | +| loss/actor | -340 | +| loss/alpha | 0.0172 | +| loss/critic | 80.2 | +| penalty/lcb_mean | 2.06 | +| penalty/lcb_std | 2.67 | +| rollout_info/dynamics_penalty_mean | 0.0016 | +| rollout_info/dynamics_penalty_std | 0.0374 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.4942 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5216 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.5243 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.4987 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.4901 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.5339 +num rollout transitions: 5000, reward mean: 4.5188 +num rollout transitions: 5000, reward mean: 4.5278 +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4812 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5263 +num rollout transitions: 5000, reward mean: 4.5446 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.4949 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.4757 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.5194 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.5068 +num rollout transitions: 5000, reward mean: 4.5017 +---------------------------------------------------------------------------------- +| alpha | 0.258 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.33 | +| loss/actor | -329 | +| loss/alpha | -0.00308 | +| loss/critic | 92 | +| penalty/lcb_mean | 2.25 | +| penalty/lcb_std | 2.93 | +| rollout_info/dynamics_penalty_mean | 0.00196 | +| rollout_info/dynamics_penalty_std | 0.0419 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.4718 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.5263 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.5281 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.5307 +num rollout transitions: 5000, reward mean: 4.5120 +num rollout transitions: 5000, reward mean: 4.5227 +num rollout transitions: 5000, reward mean: 4.5060 +num rollout transitions: 5000, reward mean: 4.4679 +num rollout transitions: 5000, reward mean: 4.4436 +num rollout transitions: 5000, reward mean: 4.4851 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.4863 +num rollout transitions: 5000, reward mean: 4.4822 +num rollout transitions: 5000, reward mean: 4.4727 +num rollout transitions: 5000, reward mean: 4.4780 +num rollout transitions: 5000, reward mean: 4.5205 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.4798 +num rollout transitions: 5000, reward mean: 4.4768 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.5265 +num rollout transitions: 5000, reward mean: 4.5054 +num rollout transitions: 5000, reward mean: 4.5293 +num rollout transitions: 5000, reward mean: 4.5364 +num rollout transitions: 5000, reward mean: 4.5363 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.4824 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4809 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.4840 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.4777 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.5382 +num rollout transitions: 5000, reward mean: 4.5178 +num rollout transitions: 5000, reward mean: 4.4981 +num rollout transitions: 5000, reward mean: 4.4933 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.5084 +num rollout transitions: 5000, reward mean: 4.5174 +---------------------------------------------------------------------------------- +| alpha | 0.239 | +| eval/episode_length | 988 | +| eval/episode_length_std | 80.2 | +| eval/normalized_episode_reward | 99.3 | +| eval/normalized_episode_reward_std | 10.9 | +| loss/actor | -326 | +| loss/alpha | 0.0016 | +| loss/critic | 93.7 | +| penalty/lcb_mean | 2.27 | +| penalty/lcb_std | 3.02 | +| rollout_info/dynamics_penalty_mean | 0.0019 | +| rollout_info/dynamics_penalty_std | 0.0411 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.4757 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.5260 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5143 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.4751 +num rollout transitions: 5000, reward mean: 4.5187 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.5252 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5049 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.4643 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5292 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5039 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.5284 +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.4708 +num rollout transitions: 5000, reward mean: 4.5183 +num rollout transitions: 5000, reward mean: 4.5257 +num rollout transitions: 5000, reward mean: 4.4802 +num rollout transitions: 5000, reward mean: 4.5146 +num rollout transitions: 5000, reward mean: 4.5149 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5436 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.5366 +num rollout transitions: 5000, reward mean: 4.5240 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.4940 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5452 +num rollout transitions: 5000, reward mean: 4.4983 +----------------------------------------------------------------------------------- +| alpha | 0.234 | +| eval/episode_length | 964 | +| eval/episode_length_std | 159 | +| eval/normalized_episode_reward | 96.4 | +| eval/normalized_episode_reward_std | 22.6 | +| loss/actor | -318 | +| loss/alpha | -0.000591 | +| loss/critic | 86 | +| penalty/lcb_mean | 2.18 | +| penalty/lcb_std | 2.82 | +| rollout_info/dynamics_penalty_mean | 0.00196 | +| rollout_info/dynamics_penalty_std | 0.0424 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2300000 | +----------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4715 +num rollout transitions: 5000, reward mean: 4.5050 +num rollout transitions: 5000, reward mean: 4.5069 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5035 +num rollout transitions: 5000, reward mean: 4.5124 +num rollout transitions: 5000, reward mean: 4.4509 +num rollout transitions: 5000, reward mean: 4.5494 +num rollout transitions: 5000, reward mean: 4.5418 +num rollout transitions: 5000, reward mean: 4.5071 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.4832 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.5324 +num rollout transitions: 5000, reward mean: 4.5140 +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.5303 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.4983 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4781 +num rollout transitions: 5000, reward mean: 4.4617 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.5065 +num rollout transitions: 5000, reward mean: 4.5027 +num rollout transitions: 5000, reward mean: 4.5178 +num rollout transitions: 5000, reward mean: 4.4731 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.5249 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.4678 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.4799 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.4806 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.4946 +num rollout transitions: 5000, reward mean: 4.5277 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5010 +---------------------------------------------------------------------------------- +| alpha | 0.243 | +| eval/episode_length | 992 | +| eval/episode_length_std | 47.3 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 5.76 | +| loss/actor | -316 | +| loss/alpha | 0.00747 | +| loss/critic | 100 | +| penalty/lcb_mean | 2.39 | +| penalty/lcb_std | 3.09 | +| rollout_info/dynamics_penalty_mean | 0.00192 | +| rollout_info/dynamics_penalty_std | 0.0415 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.4880 +num rollout transitions: 5000, reward mean: 4.4819 +num rollout transitions: 5000, reward mean: 4.5271 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5021 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.5022 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.5232 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.4908 +num rollout transitions: 5000, reward mean: 4.5262 +num rollout transitions: 5000, reward mean: 4.4990 +num rollout transitions: 5000, reward mean: 4.5200 +num rollout transitions: 5000, reward mean: 4.5423 +num rollout transitions: 5000, reward mean: 4.4689 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.5331 +num rollout transitions: 5000, reward mean: 4.5259 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4938 +num rollout transitions: 5000, reward mean: 4.4912 +num rollout transitions: 5000, reward mean: 4.5198 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.4955 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.5197 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.4581 +num rollout transitions: 5000, reward mean: 4.4936 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4735 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.5264 +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.5134 +---------------------------------------------------------------------------------- +| alpha | 0.261 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 0.569 | +| loss/actor | -329 | +| loss/alpha | -0.00333 | +| loss/critic | 115 | +| penalty/lcb_mean | 2.56 | +| penalty/lcb_std | 3.23 | +| rollout_info/dynamics_penalty_mean | 0.00306 | +| rollout_info/dynamics_penalty_std | 0.0524 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4735 +num rollout transitions: 5000, reward mean: 4.4832 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5031 +num rollout transitions: 5000, reward mean: 4.5079 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.4793 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.4904 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.5377 +num rollout transitions: 5000, reward mean: 4.4765 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5220 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.5150 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.5227 +num rollout transitions: 5000, reward mean: 4.5271 +num rollout transitions: 5000, reward mean: 4.5273 +num rollout transitions: 5000, reward mean: 4.5319 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5151 +num rollout transitions: 5000, reward mean: 4.5058 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.5241 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.5573 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.5178 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.5340 +num rollout transitions: 5000, reward mean: 4.5291 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.5182 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5038 +num rollout transitions: 5000, reward mean: 4.5391 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.5033 +---------------------------------------------------------------------------------- +| alpha | 0.265 | +| eval/episode_length | 925 | +| eval/episode_length_std | 225 | +| eval/normalized_episode_reward | 99.9 | +| eval/normalized_episode_reward_std | 26.9 | +| loss/actor | -330 | +| loss/alpha | 0.00335 | +| loss/critic | 119 | +| penalty/lcb_mean | 2.62 | +| penalty/lcb_std | 3.34 | +| rollout_info/dynamics_penalty_mean | 0.00196 | +| rollout_info/dynamics_penalty_std | 0.0415 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5226 +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.5165 +num rollout transitions: 5000, reward mean: 4.5554 +num rollout transitions: 5000, reward mean: 4.4925 +num rollout transitions: 5000, reward mean: 4.5225 +num rollout transitions: 5000, reward mean: 4.5228 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5286 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.5426 +num rollout transitions: 5000, reward mean: 4.5469 +num rollout transitions: 5000, reward mean: 4.5354 +num rollout transitions: 5000, reward mean: 4.5269 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.5186 +num rollout transitions: 5000, reward mean: 4.5413 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.4945 +num rollout transitions: 5000, reward mean: 4.5156 +num rollout transitions: 5000, reward mean: 4.5041 +num rollout transitions: 5000, reward mean: 4.5320 +num rollout transitions: 5000, reward mean: 4.5252 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5086 +num rollout transitions: 5000, reward mean: 4.4655 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.5185 +num rollout transitions: 5000, reward mean: 4.5051 +num rollout transitions: 5000, reward mean: 4.4891 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.5171 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5219 +num rollout transitions: 5000, reward mean: 4.5080 +num rollout transitions: 5000, reward mean: 4.4992 +num rollout transitions: 5000, reward mean: 4.5005 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.5277 +---------------------------------------------------------------------------------- +| alpha | 0.252 | +| eval/episode_length | 253 | +| eval/episode_length_std | 109 | +| eval/normalized_episode_reward | 20.4 | +| eval/normalized_episode_reward_std | 12.3 | +| loss/actor | -345 | +| loss/alpha | -0.00872 | +| loss/critic | 124 | +| penalty/lcb_mean | 2.71 | +| penalty/lcb_std | 3.45 | +| rollout_info/dynamics_penalty_mean | 0.00338 | +| rollout_info/dynamics_penalty_std | 0.0545 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5500 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5283 +num rollout transitions: 5000, reward mean: 4.5137 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.5042 +num rollout transitions: 5000, reward mean: 4.5190 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.5012 +num rollout transitions: 5000, reward mean: 4.5115 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.5292 +num rollout transitions: 5000, reward mean: 4.5167 +num rollout transitions: 5000, reward mean: 4.5003 +num rollout transitions: 5000, reward mean: 4.4844 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.5202 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.4934 +num rollout transitions: 5000, reward mean: 4.5179 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.4903 +num rollout transitions: 5000, reward mean: 4.4921 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.4756 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.4730 +num rollout transitions: 5000, reward mean: 4.4904 +num rollout transitions: 5000, reward mean: 4.5123 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.5196 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.4845 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.5233 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5337 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.4742 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.4674 +num rollout transitions: 5000, reward mean: 4.4734 +num rollout transitions: 5000, reward mean: 4.5157 +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.5029 +---------------------------------------------------------------------------------- +| alpha | 0.264 | +| eval/episode_length | 25.7 | +| eval/episode_length_std | 2.45 | +| eval/normalized_episode_reward | 0.0108 | +| eval/normalized_episode_reward_std | 0.035 | +| loss/actor | -347 | +| loss/alpha | 0.0133 | +| loss/critic | 133 | +| penalty/lcb_mean | 2.79 | +| penalty/lcb_std | 3.49 | +| rollout_info/dynamics_penalty_mean | 0.00201 | +| rollout_info/dynamics_penalty_std | 0.0427 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4830 +num rollout transitions: 5000, reward mean: 4.4839 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.5287 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.5281 +num rollout transitions: 5000, reward mean: 4.5212 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.4911 +num rollout transitions: 5000, reward mean: 4.4886 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.4882 +num rollout transitions: 5000, reward mean: 4.4870 +num rollout transitions: 5000, reward mean: 4.5073 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.4773 +num rollout transitions: 5000, reward mean: 4.4731 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5011 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.4824 +num rollout transitions: 5000, reward mean: 4.5209 +num rollout transitions: 5000, reward mean: 4.4963 +num rollout transitions: 5000, reward mean: 4.4865 +num rollout transitions: 5000, reward mean: 4.5119 +num rollout transitions: 5000, reward mean: 4.5248 +num rollout transitions: 5000, reward mean: 4.5310 +num rollout transitions: 5000, reward mean: 4.5153 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.4909 +num rollout transitions: 5000, reward mean: 4.4768 +num rollout transitions: 5000, reward mean: 4.4776 +num rollout transitions: 5000, reward mean: 4.4878 +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.5206 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.5240 +num rollout transitions: 5000, reward mean: 4.5145 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.4968 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.4922 +num rollout transitions: 5000, reward mean: 4.4742 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.5174 +---------------------------------------------------------------------------------- +| alpha | 0.293 | +| eval/episode_length | 189 | +| eval/episode_length_std | 81.6 | +| eval/normalized_episode_reward | 12.1 | +| eval/normalized_episode_reward_std | 9.19 | +| loss/actor | -345 | +| loss/alpha | 0.00353 | +| loss/critic | 147 | +| penalty/lcb_mean | 2.96 | +| penalty/lcb_std | 3.62 | +| rollout_info/dynamics_penalty_mean | 0.00306 | +| rollout_info/dynamics_penalty_std | 0.0525 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5294 +num rollout transitions: 5000, reward mean: 4.5297 +num rollout transitions: 5000, reward mean: 4.5216 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.4907 +num rollout transitions: 5000, reward mean: 4.5398 +num rollout transitions: 5000, reward mean: 4.4549 +num rollout transitions: 5000, reward mean: 4.4775 +num rollout transitions: 5000, reward mean: 4.4791 +num rollout transitions: 5000, reward mean: 4.4628 +num rollout transitions: 5000, reward mean: 4.5499 +num rollout transitions: 5000, reward mean: 4.4881 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.4991 +num rollout transitions: 5000, reward mean: 4.5063 +num rollout transitions: 5000, reward mean: 4.4794 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.4847 +num rollout transitions: 5000, reward mean: 4.4941 +num rollout transitions: 5000, reward mean: 4.4693 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.4729 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.4774 +num rollout transitions: 5000, reward mean: 4.5187 +num rollout transitions: 5000, reward mean: 4.5236 +num rollout transitions: 5000, reward mean: 4.5199 +num rollout transitions: 5000, reward mean: 4.4811 +num rollout transitions: 5000, reward mean: 4.4951 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.5292 +num rollout transitions: 5000, reward mean: 4.5019 +num rollout transitions: 5000, reward mean: 4.5189 +num rollout transitions: 5000, reward mean: 4.5105 +num rollout transitions: 5000, reward mean: 4.4814 +num rollout transitions: 5000, reward mean: 4.5141 +num rollout transitions: 5000, reward mean: 4.5082 +num rollout transitions: 5000, reward mean: 4.4857 +num rollout transitions: 5000, reward mean: 4.5103 +num rollout transitions: 5000, reward mean: 4.4795 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.4757 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.4808 +num rollout transitions: 5000, reward mean: 4.4864 +num rollout transitions: 5000, reward mean: 4.5064 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.5030 +---------------------------------------------------------------------------------- +| alpha | 0.279 | +| eval/episode_length | 776 | +| eval/episode_length_std | 325 | +| eval/normalized_episode_reward | 70.6 | +| eval/normalized_episode_reward_std | 31.4 | +| loss/actor | -348 | +| loss/alpha | 0.00557 | +| loss/critic | 189 | +| penalty/lcb_mean | 3.03 | +| penalty/lcb_std | 4.06 | +| rollout_info/dynamics_penalty_mean | 0.00367 | +| rollout_info/dynamics_penalty_std | 0.0569 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5113 +num rollout transitions: 5000, reward mean: 4.4866 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.4973 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.4957 +num rollout transitions: 5000, reward mean: 4.5285 +num rollout transitions: 5000, reward mean: 4.4707 +num rollout transitions: 5000, reward mean: 4.5040 +num rollout transitions: 5000, reward mean: 4.5343 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5174 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.4952 +num rollout transitions: 5000, reward mean: 4.4584 +num rollout transitions: 5000, reward mean: 4.4856 +num rollout transitions: 5000, reward mean: 4.4890 +num rollout transitions: 5000, reward mean: 4.4906 +num rollout transitions: 5000, reward mean: 4.5025 +num rollout transitions: 5000, reward mean: 4.4895 +num rollout transitions: 5000, reward mean: 4.5013 +num rollout transitions: 5000, reward mean: 4.5014 +num rollout transitions: 5000, reward mean: 4.4799 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.4748 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.4972 +num rollout transitions: 5000, reward mean: 4.5195 +num rollout transitions: 5000, reward mean: 4.4956 +num rollout transitions: 5000, reward mean: 4.4463 +num rollout transitions: 5000, reward mean: 4.5327 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.4782 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.5223 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.5116 +num rollout transitions: 5000, reward mean: 4.5272 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.4658 +num rollout transitions: 5000, reward mean: 4.5230 +num rollout transitions: 5000, reward mean: 4.4967 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.5104 +---------------------------------------------------------------------------------- +| alpha | 0.272 | +| eval/episode_length | 948 | +| eval/episode_length_std | 141 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 16.5 | +| loss/actor | -344 | +| loss/alpha | -0.00868 | +| loss/critic | 136 | +| penalty/lcb_mean | 2.83 | +| penalty/lcb_std | 3.56 | +| rollout_info/dynamics_penalty_mean | 0.00735 | +| rollout_info/dynamics_penalty_std | 0.0813 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4982 +num rollout transitions: 5000, reward mean: 4.5032 +num rollout transitions: 5000, reward mean: 4.5162 +num rollout transitions: 5000, reward mean: 4.5501 +num rollout transitions: 5000, reward mean: 4.5210 +num rollout transitions: 5000, reward mean: 4.4944 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.4804 +num rollout transitions: 5000, reward mean: 4.4920 +num rollout transitions: 5000, reward mean: 4.5228 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5163 +num rollout transitions: 5000, reward mean: 4.5176 +num rollout transitions: 5000, reward mean: 4.4993 +num rollout transitions: 5000, reward mean: 4.5100 +num rollout transitions: 5000, reward mean: 4.4959 +num rollout transitions: 5000, reward mean: 4.5257 +num rollout transitions: 5000, reward mean: 4.4820 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.4843 +num rollout transitions: 5000, reward mean: 4.4821 +num rollout transitions: 5000, reward mean: 4.4606 +num rollout transitions: 5000, reward mean: 4.4998 +num rollout transitions: 5000, reward mean: 4.4876 +num rollout transitions: 5000, reward mean: 4.4715 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.5056 +num rollout transitions: 5000, reward mean: 4.5055 +num rollout transitions: 5000, reward mean: 4.5290 +num rollout transitions: 5000, reward mean: 4.5342 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.4984 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.5048 +num rollout transitions: 5000, reward mean: 4.5301 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5018 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.5227 +num rollout transitions: 5000, reward mean: 4.4914 +num rollout transitions: 5000, reward mean: 4.5098 +num rollout transitions: 5000, reward mean: 4.5280 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.5288 +num rollout transitions: 5000, reward mean: 4.5330 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.4931 +num rollout transitions: 5000, reward mean: 4.5135 +num rollout transitions: 5000, reward mean: 4.5510 +---------------------------------------------------------------------------------- +| alpha | 0.26 | +| eval/episode_length | 948 | +| eval/episode_length_std | 136 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 17.3 | +| loss/actor | -323 | +| loss/alpha | -0.00337 | +| loss/critic | 134 | +| penalty/lcb_mean | 2.82 | +| penalty/lcb_std | 3.37 | +| rollout_info/dynamics_penalty_mean | 0.00243 | +| rollout_info/dynamics_penalty_std | 0.0465 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.5028 +num rollout transitions: 5000, reward mean: 4.5201 +num rollout transitions: 5000, reward mean: 4.5010 +num rollout transitions: 5000, reward mean: 4.5083 +num rollout transitions: 5000, reward mean: 4.5224 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.4882 +num rollout transitions: 5000, reward mean: 4.5030 +num rollout transitions: 5000, reward mean: 4.4880 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4996 +num rollout transitions: 5000, reward mean: 4.4628 +num rollout transitions: 5000, reward mean: 4.4961 +num rollout transitions: 5000, reward mean: 4.5019 +num rollout transitions: 5000, reward mean: 4.4880 +num rollout transitions: 5000, reward mean: 4.5065 +num rollout transitions: 5000, reward mean: 4.5111 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5222 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.5398 +num rollout transitions: 5000, reward mean: 4.4646 +num rollout transitions: 5000, reward mean: 4.4813 +num rollout transitions: 5000, reward mean: 4.4926 +num rollout transitions: 5000, reward mean: 4.5330 +num rollout transitions: 5000, reward mean: 4.5298 +num rollout transitions: 5000, reward mean: 4.5011 +num rollout transitions: 5000, reward mean: 4.4832 +num rollout transitions: 5000, reward mean: 4.4985 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.5231 +num rollout transitions: 5000, reward mean: 4.4975 +num rollout transitions: 5000, reward mean: 4.5487 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.4783 +num rollout transitions: 5000, reward mean: 4.5142 +num rollout transitions: 5000, reward mean: 4.5449 +num rollout transitions: 5000, reward mean: 4.5104 +num rollout transitions: 5000, reward mean: 4.5286 +num rollout transitions: 5000, reward mean: 4.4695 +num rollout transitions: 5000, reward mean: 4.5215 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.5304 +num rollout transitions: 5000, reward mean: 4.5161 +num rollout transitions: 5000, reward mean: 4.5211 +num rollout transitions: 5000, reward mean: 4.5216 +num rollout transitions: 5000, reward mean: 4.5114 +num rollout transitions: 5000, reward mean: 4.4786 +num rollout transitions: 5000, reward mean: 4.5239 +---------------------------------------------------------------------------------- +| alpha | 0.284 | +| eval/episode_length | 992 | +| eval/episode_length_std | 83.1 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 9.03 | +| loss/actor | -352 | +| loss/alpha | 0.0197 | +| loss/critic | 168 | +| penalty/lcb_mean | 3.17 | +| penalty/lcb_std | 3.89 | +| rollout_info/dynamics_penalty_mean | 0.00346 | +| rollout_info/dynamics_penalty_std | 0.0568 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5134 +num rollout transitions: 5000, reward mean: 4.4898 +num rollout transitions: 5000, reward mean: 4.4877 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.5281 +num rollout transitions: 5000, reward mean: 4.4974 +num rollout transitions: 5000, reward mean: 4.5208 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5239 +num rollout transitions: 5000, reward mean: 4.5229 +num rollout transitions: 5000, reward mean: 4.5443 +num rollout transitions: 5000, reward mean: 4.5000 +num rollout transitions: 5000, reward mean: 4.4929 +num rollout transitions: 5000, reward mean: 4.4692 +num rollout transitions: 5000, reward mean: 4.4749 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.5270 +num rollout transitions: 5000, reward mean: 4.5122 +num rollout transitions: 5000, reward mean: 4.4999 +num rollout transitions: 5000, reward mean: 4.4803 +num rollout transitions: 5000, reward mean: 4.4923 +num rollout transitions: 5000, reward mean: 4.4919 +num rollout transitions: 5000, reward mean: 4.4887 +num rollout transitions: 5000, reward mean: 4.5121 +num rollout transitions: 5000, reward mean: 4.5066 +num rollout transitions: 5000, reward mean: 4.5044 +num rollout transitions: 5000, reward mean: 4.5090 +num rollout transitions: 5000, reward mean: 4.5101 +num rollout transitions: 5000, reward mean: 4.4768 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.5085 +num rollout transitions: 5000, reward mean: 4.5699 +num rollout transitions: 5000, reward mean: 4.5343 +num rollout transitions: 5000, reward mean: 4.5367 +num rollout transitions: 5000, reward mean: 4.4768 +num rollout transitions: 5000, reward mean: 4.5286 +num rollout transitions: 5000, reward mean: 4.5237 +num rollout transitions: 5000, reward mean: 4.5094 +num rollout transitions: 5000, reward mean: 4.5097 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.4859 +num rollout transitions: 5000, reward mean: 4.4970 +num rollout transitions: 5000, reward mean: 4.4964 +num rollout transitions: 5000, reward mean: 4.4982 +---------------------------------------------------------------------------------- +| alpha | 0.244 | +| eval/episode_length | 982 | +| eval/episode_length_std | 98.2 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 11.1 | +| loss/actor | -330 | +| loss/alpha | -0.0467 | +| loss/critic | 153 | +| penalty/lcb_mean | 3.02 | +| penalty/lcb_std | 3.66 | +| rollout_info/dynamics_penalty_mean | 0.00255 | +| rollout_info/dynamics_penalty_std | 0.0483 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.4750 +num rollout transitions: 5000, reward mean: 4.5310 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.4798 +num rollout transitions: 5000, reward mean: 4.4724 +num rollout transitions: 5000, reward mean: 4.4559 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.4840 +num rollout transitions: 5000, reward mean: 4.5138 +num rollout transitions: 5000, reward mean: 4.5126 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.5455 +num rollout transitions: 5000, reward mean: 4.5162 +num rollout transitions: 5000, reward mean: 4.5414 +num rollout transitions: 5000, reward mean: 4.5437 +num rollout transitions: 5000, reward mean: 4.5207 +num rollout transitions: 5000, reward mean: 4.5096 +num rollout transitions: 5000, reward mean: 4.5351 +num rollout transitions: 5000, reward mean: 4.5476 +num rollout transitions: 5000, reward mean: 4.5282 +num rollout transitions: 5000, reward mean: 4.4966 +num rollout transitions: 5000, reward mean: 4.4862 +num rollout transitions: 5000, reward mean: 4.5029 +num rollout transitions: 5000, reward mean: 4.5174 +num rollout transitions: 5000, reward mean: 4.4851 +num rollout transitions: 5000, reward mean: 4.4950 +num rollout transitions: 5000, reward mean: 4.4932 +num rollout transitions: 5000, reward mean: 4.4771 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.5378 +num rollout transitions: 5000, reward mean: 4.4944 +num rollout transitions: 5000, reward mean: 4.5241 +num rollout transitions: 5000, reward mean: 4.5078 +num rollout transitions: 5000, reward mean: 4.5188 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.4853 +num rollout transitions: 5000, reward mean: 4.4928 +num rollout transitions: 5000, reward mean: 4.5144 +num rollout transitions: 5000, reward mean: 4.5059 +num rollout transitions: 5000, reward mean: 4.5002 +num rollout transitions: 5000, reward mean: 4.4879 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5214 +num rollout transitions: 5000, reward mean: 4.5092 +num rollout transitions: 5000, reward mean: 4.4788 +num rollout transitions: 5000, reward mean: 4.4764 +num rollout transitions: 5000, reward mean: 4.4875 +num rollout transitions: 5000, reward mean: 4.5053 +num rollout transitions: 5000, reward mean: 4.5034 +num rollout transitions: 5000, reward mean: 4.5020 +---------------------------------------------------------------------------------- +| alpha | 0.266 | +| eval/episode_length | 153 | +| eval/episode_length_std | 121 | +| eval/normalized_episode_reward | 10.6 | +| eval/normalized_episode_reward_std | 14.8 | +| loss/actor | -318 | +| loss/alpha | 0.108 | +| loss/critic | 421 | +| penalty/lcb_mean | 3.11 | +| penalty/lcb_std | 4.45 | +| rollout_info/dynamics_penalty_mean | 0.00326 | +| rollout_info/dynamics_penalty_std | 0.0546 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.5166 +num rollout transitions: 5000, reward mean: 4.5075 +num rollout transitions: 5000, reward mean: 4.4800 +num rollout transitions: 5000, reward mean: 4.5091 +num rollout transitions: 5000, reward mean: 4.4937 +num rollout transitions: 5000, reward mean: 4.4689 +num rollout transitions: 5000, reward mean: 4.4965 +num rollout transitions: 5000, reward mean: 4.4849 +num rollout transitions: 5000, reward mean: 4.4880 +num rollout transitions: 5000, reward mean: 4.4873 +num rollout transitions: 5000, reward mean: 4.4977 +num rollout transitions: 5000, reward mean: 4.4918 +num rollout transitions: 5000, reward mean: 4.4902 +num rollout transitions: 5000, reward mean: 4.4889 +num rollout transitions: 5000, reward mean: 4.4978 +num rollout transitions: 5000, reward mean: 4.5132 +num rollout transitions: 5000, reward mean: 4.4916 +num rollout transitions: 5000, reward mean: 4.5315 +num rollout transitions: 5000, reward mean: 4.5191 +num rollout transitions: 5000, reward mean: 4.5062 +num rollout transitions: 5000, reward mean: 4.5158 +num rollout transitions: 5000, reward mean: 4.5147 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5129 +num rollout transitions: 5000, reward mean: 4.5273 +num rollout transitions: 5000, reward mean: 4.5006 +num rollout transitions: 5000, reward mean: 4.5033 +num rollout transitions: 5000, reward mean: 4.5067 +num rollout transitions: 5000, reward mean: 4.5108 +num rollout transitions: 5000, reward mean: 4.5228 +num rollout transitions: 5000, reward mean: 4.4979 +num rollout transitions: 5000, reward mean: 4.5216 +num rollout transitions: 5000, reward mean: 4.5155 +num rollout transitions: 5000, reward mean: 4.4924 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.5244 +num rollout transitions: 5000, reward mean: 4.5127 +num rollout transitions: 5000, reward mean: 4.5008 +num rollout transitions: 5000, reward mean: 4.5255 +num rollout transitions: 5000, reward mean: 4.4691 +num rollout transitions: 5000, reward mean: 4.5112 +num rollout transitions: 5000, reward mean: 4.5339 +num rollout transitions: 5000, reward mean: 4.4935 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.5011 +num rollout transitions: 5000, reward mean: 4.5099 +num rollout transitions: 5000, reward mean: 4.4835 +num rollout transitions: 5000, reward mean: 4.4986 +num rollout transitions: 5000, reward mean: 4.5221 +---------------------------------------------------------------------------------- +| alpha | 0.23 | +| eval/episode_length | 136 | +| eval/episode_length_std | 4.6 | +| eval/normalized_episode_reward | 8.51 | +| eval/normalized_episode_reward_std | 0.403 | +| loss/actor | -317 | +| loss/alpha | 0.0365 | +| loss/critic | 114 | +| penalty/lcb_mean | 2.7 | +| penalty/lcb_std | 3.16 | +| rollout_info/dynamics_penalty_mean | 0.00406 | +| rollout_info/dynamics_penalty_std | 0.0617 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 5000, reward mean: 4.5046 +num rollout transitions: 5000, reward mean: 4.5007 +num rollout transitions: 5000, reward mean: 4.5117 +num rollout transitions: 5000, reward mean: 4.5043 +num rollout transitions: 5000, reward mean: 4.4836 +num rollout transitions: 5000, reward mean: 4.4894 +num rollout transitions: 5000, reward mean: 4.5037 +num rollout transitions: 5000, reward mean: 4.5131 +num rollout transitions: 5000, reward mean: 4.5250 +num rollout transitions: 5000, reward mean: 4.5020 +num rollout transitions: 5000, reward mean: 4.5173 +num rollout transitions: 5000, reward mean: 4.4897 +num rollout transitions: 5000, reward mean: 4.5125 +num rollout transitions: 5000, reward mean: 4.4915 +num rollout transitions: 5000, reward mean: 4.4997 +num rollout transitions: 5000, reward mean: 4.5026 +num rollout transitions: 5000, reward mean: 4.5148 +num rollout transitions: 5000, reward mean: 4.5460 +num rollout transitions: 5000, reward mean: 4.4760 +num rollout transitions: 5000, reward mean: 4.4874 +num rollout transitions: 5000, reward mean: 4.4905 +num rollout transitions: 5000, reward mean: 4.5258 +num rollout transitions: 5000, reward mean: 4.5393 +num rollout transitions: 5000, reward mean: 4.5023 +num rollout transitions: 5000, reward mean: 4.5322 +num rollout transitions: 5000, reward mean: 4.5045 +num rollout transitions: 5000, reward mean: 4.5089 +num rollout transitions: 5000, reward mean: 4.5001 +num rollout transitions: 5000, reward mean: 4.4927 +num rollout transitions: 5000, reward mean: 4.5107 +num rollout transitions: 5000, reward mean: 4.4948 +num rollout transitions: 5000, reward mean: 4.5225 +num rollout transitions: 5000, reward mean: 4.4989 +num rollout transitions: 5000, reward mean: 4.5109 +num rollout transitions: 5000, reward mean: 4.5074 +num rollout transitions: 5000, reward mean: 4.5152 +num rollout transitions: 5000, reward mean: 4.4947 +num rollout transitions: 5000, reward mean: 4.5316 +num rollout transitions: 5000, reward mean: 4.4913 +num rollout transitions: 5000, reward mean: 4.5016 +num rollout transitions: 5000, reward mean: 4.5440 +num rollout transitions: 5000, reward mean: 4.5015 +num rollout transitions: 5000, reward mean: 4.5260 +num rollout transitions: 5000, reward mean: 4.5218 +num rollout transitions: 5000, reward mean: 4.4880 +num rollout transitions: 5000, reward mean: 4.5081 +num rollout transitions: 5000, reward mean: 4.5077 +num rollout transitions: 5000, reward mean: 4.5299 +num rollout transitions: 5000, reward mean: 4.5024 +num rollout transitions: 5000, reward mean: 4.5207 +---------------------------------------------------------------------------------- +| alpha | 0.26 | +| eval/episode_length | 133 | +| eval/episode_length_std | 1.12 | +| eval/normalized_episode_reward | 8.27 | +| eval/normalized_episode_reward_std | 0.0553 | +| loss/actor | -280 | +| loss/alpha | 0.0228 | +| loss/critic | 103 | +| penalty/lcb_mean | 2.6 | +| penalty/lcb_std | 2.95 | +| rollout_info/dynamics_penalty_mean | 0.00387 | +| rollout_info/dynamics_penalty_std | 0.0603 | +| rollout_info/num_transitions | 5e+03 | +| rollout_info/reward_mean | 4.51 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 48907.26s diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193338/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193338/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193338/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193338/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..bd2c102a3b4cdacb815f7838deaee108047a606f --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193338/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_neuralode.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 456, "device": "cuda:6", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_456×tamp_26-0112-030559/model", "reward_penalty_coef": 0.3, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/neuralODE", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 5000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "neuralODE", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "neuralODE"} diff --git a/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193338/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193338/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..bd57aa5ee4f97c62f7df10382302586867c88f74 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/neuralODE/seed_456×tamp_26-0506-193338/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.27445982723131757,58.76,3.611426311030034,0.048243503492669045,-0.009801653650806586,-299.0151374817539,-1.7365832626876758,25.23035614417292,1.1004732710011802,1.4749216053414942,0.016544376492765495,0.10775360212766559,5000.0,4.506422794791187,50000 +0.15650416074305715,97.51,23.39892946269124,4.906340921383183,2.227525861592355,-420.5790685424861,-0.04408288361728065,32.69792118272739,1.2277664351248334,1.8216326573968091,0.0026458314696885645,0.04785291277989041,5000.0,4.512616795554159,100000 +0.15422011206895037,178.04,3.802420281873112,10.534976748620283,0.33359354185981355,-422.8249055377241,0.005133900864869216,30.230876207656692,1.2155333330559623,1.856482727460861,0.003477149940528907,0.05389121436865566,5000.0,4.516083530865981,150000 +0.16929313008785313,124.49,10.099995049503736,5.318993936067851,0.913986265306235,-422.31953304809645,0.02000332812935151,34.577267468319334,1.302212422840591,2.0087742873442043,0.003411602893956007,0.054532315468080635,5000.0,4.5125705086288646,200000 +0.1623209073817733,98.78,36.83573808137962,5.281104368556507,2.692599067507329,-432.2831537976045,-0.020250362604744825,31.615563918705522,1.2561060014820153,1.8777683100104177,0.0028633924687169483,0.04987384358777723,5000.0,4.512650641442235,250000 +0.1588302907237416,120.89,9.596765079963143,5.3458243958272265,0.4395613144073724,-428.31250946594116,0.01405265935808415,31.63010903144831,1.2890754461395708,1.901991405185455,0.0030344801900070153,0.052413738824605566,5000.0,4.512331942095282,300000 +0.1675838469815239,77.13,1.8366001197865587,4.052536951971254,0.09342035365965555,-405.25411756713584,-0.0007153304792568922,35.538524167480354,1.3496274845588305,1.9959538223349969,0.002676066423216835,0.04904703097345298,5000.0,4.518792244217951,350000 +0.16781114623785115,118.14,59.696401901622174,7.262737420410077,6.253344087234367,-395.62145939574754,0.008893691777810525,35.64147666934962,1.355997879700645,1.9266114126813514,0.00418164946752973,0.06042274425351135,5000.0,4.5143571833498735,400000 +0.16426335055410798,183.37,13.813511501424975,12.221449866721214,1.4107252761245954,-392.3708710040227,-0.010246583940573168,36.145931464081364,1.3777643044686307,1.9736141508936935,0.0028608828081640534,0.05059078303061409,5000.0,4.5107638179991145,450000 +0.16806549064517232,345.16,94.6666488262894,30.392662677368044,10.950237816036399,-387.6006827252239,-0.004095645905547096,40.813752370739024,1.4151086788880705,2.176962863756407,0.0028141174331270144,0.050378854020420735,5000.0,4.509120090644186,500000 +0.17314580733597182,996.4,20.49829261182502,94.55071908329056,1.9946364481972505,-384.7655327648939,0.02091965269543236,43.31751118783952,1.3980934320437985,2.184228104467407,0.0024165062968907877,0.046323111683430424,5000.0,4.510179529840453,550000 +0.17753058204054828,438.43,370.1165561009126,39.81745839433991,39.966529857235564,-371.74316504455345,-0.008358159860670681,50.77944208541863,1.483626173760899,2.3102804792380622,0.0024143541997203137,0.046956736033164845,5000.0,4.514777019082042,600000 +0.1735239286357153,12.02,0.13999999999999999,-0.25478400167392345,-0.028764003315541357,-360.98112014831673,-0.0024225035136936903,51.405305866622776,1.5121711923480137,2.3910273299669984,0.0030053048236356123,0.052010964279053544,5000.0,4.504731006956533,650000 +0.19667688758700952,90.2,11.55768142838346,3.670266963852029,0.7647976600435347,-360.19934830994305,-0.008378687376975726,51.06705545219393,1.5644886018347872,2.291096643378748,0.002748201054359262,0.04907216835401662,5000.0,4.510931269303889,700000 +0.1919497848027974,809.97,151.41687191327128,84.10920472486694,18.12919943555399,-353.6609963983171,0.024154111461620824,50.62958808708212,1.538783950321688,2.2860163400197058,0.0022109941045008593,0.043989732218015026,5000.0,4.5061402291468164,750000 +0.18526475465327408,101.01,3.745650811274324,4.919399574020262,0.2450892406881774,-352.94682039916995,-0.03666798772431903,49.01518828136453,1.5072728182578106,2.299382735422802,0.0019978686243528497,0.04147755169167675,5000.0,4.503343829088896,800000 +0.1857965996909151,921.52,218.40821779411135,88.47755597614214,22.583146986367193,-358.75901142395173,0.03158931383354604,51.631927884826986,1.5389202019977581,2.4136771587610157,0.002180633040687534,0.04415321526165574,5000.0,4.504522137354429,850000 +0.19261602549999796,1000.0,0.0,98.9467955016529,0.872095057665178,-356.0075305108651,0.0034196057696268644,55.64134660808558,1.5697047018384944,2.4390386736321403,0.0017553430520510299,0.03951620461715857,5000.0,4.503696662312456,900000 +0.20406064885765327,997.75,22.38721733489895,101.04163596868865,2.794206108958864,-339.22578455505754,-0.0013035734269767688,59.648059092102166,1.6257606876969353,2.466291263433671,0.0016258343608053394,0.03779337749514925,5000.0,4.495086132314637,950000 +0.1923651621380427,179.42,6.4887286890422535,11.280545007136924,0.6401099553695705,-340.03854445984047,-0.014415794693641315,61.27254649555195,1.6615057304787653,2.522667735199917,0.0017061321914382279,0.039078621639983935,5000.0,4.503740728951041,1000000 +0.20309797072201902,168.46,84.41485888159738,10.188265442936837,8.226396408187965,-335.10693101134933,0.0058566226462273925,60.7285388907999,1.6514142594814352,2.5206640251040766,0.0019645370887584578,0.04172511164641498,5000.0,4.503401291020906,1050000 +0.20711749149709827,874.81,195.46108026919322,91.81930204767654,22.117133192245912,-338.1762841949507,0.006964028028435952,60.154094173660155,1.655164017555711,2.458517143092108,0.001451180902928114,0.03556084058924875,5000.0,4.50250156673801,1100000 +0.20193127109825743,934.0,152.46415972286732,102.34174483419368,17.718314354511055,-339.4337467163065,-0.00733189297100532,66.87559799564329,1.7278595274162427,2.6420127394628357,0.0017082465765761888,0.039297781556473656,5000.0,4.497677327223605,1150000 +0.20770911349326132,198.16,8.859706541415466,14.219606099433598,0.8182456643105805,-329.1822522552502,0.006530123744774513,63.558464619025756,1.7013213634657909,2.473683260669745,0.0018870845290124413,0.040971739678265326,5000.0,4.49025703076893,1200000 +0.2111136317056401,266.65,183.012151235922,23.16282036847722,20.736655370254773,-337.662706586306,0.0017401144363357914,68.44027436004663,1.7904654431509734,2.6429161606621547,0.0016170079413279884,0.038174255138630035,5000.0,4.496783833592797,1250000 +0.21884165040939854,930.76,161.26661898855573,96.70334073139905,18.070587333039384,-338.89614486327866,0.0036803869894147636,67.87744378074646,1.8043905882883151,2.6477286016965014,0.0012784025100742004,0.0330213268885102,5000.0,4.497368208864333,1300000 +0.2164278635704534,1000.0,0.0,102.32466215273178,0.9224461374200648,-336.2265158544897,-0.011744169391207374,71.49133045207807,1.7767541059637042,2.6980511859035383,0.001382069980908185,0.035220320900465084,5000.0,4.501974271027143,1350000 +0.22322152300685716,181.53,84.1897208689992,2.2113088517129547,8.729237404833489,-338.4607023730511,0.010865496915019942,70.74643342021865,1.7667818131756718,2.6410140906810846,0.0016018505274867632,0.037398751617399424,5000.0,4.501715851598082,1400000 +0.2172004560181481,586.24,406.3389501389203,54.96881383449902,44.299650773155015,-338.1098791625907,-0.003734454952217578,65.8448103142924,1.7612564051985635,2.5779158208609516,0.001579043241105974,0.03749323489332815,5000.0,4.5022586286692166,1450000 +0.22388580485969845,181.31,169.09285585145224,13.41173019020068,19.410956888841813,-345.06513647094124,-0.008818590951413009,68.80350349918413,1.7670145556760048,2.7121559936713924,0.0017849585101259876,0.039678124895787224,5000.0,4.514931248618963,1500000 +0.2118772602996253,938.15,160.81059511114307,90.25229035630775,18.349214709612895,-337.43876251159475,0.01779167991150156,71.00405672870598,1.806435787658678,2.7082205743097996,0.0020607750533372157,0.04281261244443501,5000.0,4.504895394246941,1550000 +0.2181262243002677,526.71,294.72405042683573,49.370841175548776,33.46635146725318,-334.3829792279052,-0.007575648148134578,72.88061548618242,1.8544740760541114,2.7542597748899333,0.0024107731408178814,0.04625075693364402,5000.0,4.505316956076499,1600000 +0.217588140442372,363.47,288.16434390812475,31.81239759677012,33.064770451125135,-334.58278222839283,-0.010374282600637403,81.79384585922296,2.0326292059302196,2.834341615288206,0.0016948967265170065,0.03861897071791802,5000.0,4.499930792478183,1650000 +0.21980553735285963,995.25,40.41890028192257,109.23918853008759,4.931239741415507,-341.9980779797331,0.013276147859170757,83.48724269504459,2.0398335444068962,2.9242227972435972,0.002589364539585076,0.048855150403440045,5000.0,4.511393831385185,1700000 +0.23757179518073498,14.0,0.0,-0.15495809506792094,-0.03328171141064946,-325.2813870831332,0.012330028834827233,81.51522989952053,1.992931924929634,2.7667958010029796,0.0021521885623827113,0.04388820531241298,5000.0,4.504544863580253,1750000 +0.2359071236985909,819.32,328.03712838640683,78.88372773941319,34.1765624374452,-322.6341194458033,-0.009137914705462396,81.06138561691297,2.0496942868804617,2.800777703273296,0.002400363552669063,0.046288988519613515,5000.0,4.498642883727656,1800000 +0.21133186813890933,924.8,206.13034711075417,99.32027369049925,24.345625184071384,-327.5586957080068,-0.004205082751661591,76.34736379619623,1.9326807900524254,2.795217812592908,0.002117620773510075,0.042992228621477856,5000.0,4.506194807539097,1850000 +0.22438187278896563,1000.0,0.0,106.57522290968792,0.980756215915922,-327.893611049197,0.006520127348415643,80.30106839588214,2.032934595522881,2.770666065533168,0.0018806295393425988,0.04110568098127416,5000.0,4.5103012902930235,1900000 +0.22456437467992407,683.35,365.86826522670697,70.64550271556986,43.41031645900134,-325.26317810119224,-0.007646453250031836,83.18163436401238,2.043229052300466,2.841058218684158,0.0019396589394523763,0.041929612489462184,5000.0,4.5112146264749144,1950000 +0.2420052423292386,150.66,8.505551128527769,10.452637393110383,0.8485859744590162,-327.09882869141074,0.008410631835255408,88.01346597740189,2.1572124127315986,2.880536975097683,0.002059644815467297,0.043220196955792003,5000.0,4.505329567987219,2000000 +0.2510119550597691,162.53,14.885197345013601,9.960376367494895,1.0501091767909556,-342.08538051513847,0.01016006871948019,98.58401368362553,2.2915022180223494,3.07870070168732,0.0025345731889959435,0.04698376418581123,5000.0,4.503416374756499,2050000 +0.24430831859470042,795.44,186.69163452067156,90.92610034271577,23.676794817598225,-345.5321493341042,-0.02150462166871868,82.09517907058792,2.1132934896564066,2.757161373555661,0.0019194503184221685,0.04110543310829926,5000.0,4.512786893325062,2100000 +0.23066441280722796,909.69,212.57303192079655,102.0709033604781,26.583444190488354,-340.204861651005,0.017209447647202763,80.23080155422245,2.0585955816650414,2.668067254579096,0.0015963406975176187,0.03737461814146985,5000.0,4.511897901448632,2150000 +0.2575881913250665,1000.0,0.0,109.15176764553915,0.3296701708139256,-328.5000267102094,-0.0030762346706539757,91.96625418464775,2.2497536155152082,2.9319227814912563,0.0019584378123972566,0.04192828963109754,5000.0,4.505538781446609,2200000 +0.2388502494636194,988.47,80.15977233001604,99.2602056355522,10.895390500072148,-325.639197027589,0.001604552300293007,93.67356849266007,2.2661196474743335,3.017324496400359,0.0019016923236185679,0.04110603195727306,5000.0,4.500990680957591,2250000 +0.23404339710771915,963.7,158.92995312400993,96.38166487045156,22.590271734050514,-317.8408663769487,-0.0005913881968333703,86.02306730613867,2.1754731187820187,2.8247771274185505,0.001960995220188983,0.042399114836813585,5000.0,4.509400533394409,2300000 +0.24345945269554592,991.83,47.27706737943885,108.0364297171412,5.763065193592463,-315.5855270953358,0.007465901104714706,100.25180096054146,2.3945901330828723,3.093840776371988,0.0019231199175119402,0.04145949101069773,5000.0,4.501870783372063,2350000 +0.2614857943657142,1000.0,0.0,106.40813182359278,0.5686842567699562,-328.80749954529296,-0.003330232220292144,115.06794468559367,2.5601117583823063,3.2335736410212688,0.0030609788945307964,0.05241111666970642,5000.0,4.504609468470373,2400000 +0.2651465222534564,925.1,224.90582473559903,99.87163953375523,26.881996445310484,-330.02153336120506,0.0033544449754246827,119.3034826409159,2.6243734606719005,3.3441216859531515,0.0019586201453153044,0.04153070519730289,5000.0,4.509834636773724,2450000 +0.2521000116223081,253.45,108.76206829589073,20.36970923148039,12.313507242589756,-345.1405446398955,-0.008718636333923798,124.09144827736043,2.7054326935839637,3.454353204715245,0.0033841298725400134,0.05450755151954424,5000.0,4.512192818794149,2500000 +0.2639045401039722,25.74,2.4520195757783014,0.010816963258851314,0.03502433408967448,-347.284158260498,0.013257998036108795,133.06728742347724,2.794784156134124,3.4874146764969667,0.0020141469420162025,0.0426573111219075,5000.0,4.50308078469461,2550000 +0.2928805559521857,188.74,81.5631804186178,12.050926862294007,9.19393729555848,-344.9462132653744,0.0035269470019824506,147.4911089510329,2.9588079670643785,3.6246591988539754,0.0030555874797254788,0.052530709356446,5000.0,4.500350578840327,2600000 +0.27876142063349596,776.08,325.1994674042379,70.61817648468389,31.443290395678385,-347.610713947144,0.005568273646384389,189.056551544876,3.0251401956653865,4.064026497359277,0.003670132058171554,0.05685378465710876,5000.0,4.4978684501918815,2650000 +0.27193445696353064,947.81,140.9243552406751,103.42665785641252,16.541290820803813,-343.87032196227955,-0.008676692698784075,135.7813771832309,2.826176201767902,3.5594498445224616,0.007346259926037866,0.08133183583040589,5000.0,4.4999012833829894,2700000 +0.26012612844318084,947.75,135.7354319991652,109.53577115408059,17.30399162903028,-322.9650578381323,-0.0033740938443691053,133.88863178001296,2.8162177366995613,3.3679091687607916,0.002429414171891752,0.04648844842722824,5000.0,4.506868837907762,2750000 +0.28438583961129577,991.65,83.08145099840279,101.35901787919752,9.030603954777176,-352.33121297362845,0.019654277418479037,168.17698437134095,3.174254003405547,3.886770778169617,0.0034627121402032225,0.056781919759984444,5000.0,4.506758328959215,2800000 +0.24422323247701172,982.47,98.2191890620158,101.50859778377156,11.128161858646422,-330.4165599139401,-0.04671147626100113,153.3191535833727,3.023675412058811,3.657755361969472,0.0025481240454018122,0.048307321191242855,5000.0,4.506376444751004,2850000 +0.2658461618939039,153.2,121.01190024125725,10.55783923726498,14.766760745186838,-317.6105591833445,0.10784925686256742,420.73584277282015,3.1063564090895626,4.452084937002661,0.0032624407606148156,0.0546462336342317,5000.0,4.505186410183448,2900000 +0.23029752267003054,136.44,4.596346375111431,8.511518510441364,0.40310287175624526,-317.4019121682755,0.036450244491324425,113.94453113128584,2.7048446783518907,3.1601748928332416,0.004063114521142094,0.0616899213719981,5000.0,4.504595078381564,2950000 +0.2604423625710529,132.51,1.117989266495882,8.271633164680786,0.055298595948617746,-280.00383818359273,0.0227993541814389,103.1348184304808,2.599766097328674,2.9468097366023156,0.0038679537339210515,0.060295537623309345,5000.0,4.508033541242726,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_123×tamp_26-0318-110814/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_123×tamp_26-0318-110814/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..59944e669aa9b75f3b9253afd3cca7b457be6aff --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_123×tamp_26-0318-110814/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 3.9998 +num rollout transitions: 50000, reward mean: 4.1087 +num rollout transitions: 50000, reward mean: 4.1764 +num rollout transitions: 50000, reward mean: 4.1770 +num rollout transitions: 50000, reward mean: 4.1956 +num rollout transitions: 50000, reward mean: 4.1614 +num rollout transitions: 50000, reward mean: 4.1849 +num rollout transitions: 50000, reward mean: 4.2251 +num rollout transitions: 50000, reward mean: 4.2859 +num rollout transitions: 50000, reward mean: 4.3189 +num rollout transitions: 50000, reward mean: 4.3417 +num rollout transitions: 50000, reward mean: 4.3602 +num rollout transitions: 50000, reward mean: 4.3404 +num rollout transitions: 50000, reward mean: 4.3588 +num rollout transitions: 50000, reward mean: 4.3795 +num rollout transitions: 50000, reward mean: 4.3767 +num rollout transitions: 50000, reward mean: 4.3706 +num rollout transitions: 50000, reward mean: 4.3579 +num rollout transitions: 50000, reward mean: 4.3321 +num rollout transitions: 50000, reward mean: 4.3857 +num rollout transitions: 50000, reward mean: 4.4082 +num rollout transitions: 50000, reward mean: 4.4008 +num rollout transitions: 50000, reward mean: 4.4028 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4196 +num rollout transitions: 50000, reward mean: 4.3952 +num rollout transitions: 50000, reward mean: 4.3903 +num rollout transitions: 50000, reward mean: 4.3589 +num rollout transitions: 50000, reward mean: 4.3217 +num rollout transitions: 50000, reward mean: 4.3058 +num rollout transitions: 50000, reward mean: 4.3436 +num rollout transitions: 50000, reward mean: 4.3874 +num rollout transitions: 50000, reward mean: 4.4072 +num rollout transitions: 50000, reward mean: 4.4106 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4146 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4105 +num rollout transitions: 50000, reward mean: 4.4138 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4608 +num rollout transitions: 50000, reward mean: 4.4507 +---------------------------------------------------------------------------------- +| alpha | 0.256 | +| eval/episode_length | 100 | +| eval/episode_length_std | 4.11 | +| eval/normalized_episode_reward | 4.51 | +| eval/normalized_episode_reward_std | 0.213 | +| loss/actor | -279 | +| loss/alpha | -1.93 | +| loss/critic | 18.5 | +| penalty/lcb_mean | 0.894 | +| penalty/lcb_std | 1.21 | +| rollout_info/dynamics_penalty_mean | 0.315 | +| rollout_info/dynamics_penalty_std | 0.376 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.36 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4574 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4551 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4567 +num rollout transitions: 50000, reward mean: 4.4503 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4541 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4135 +num rollout transitions: 50000, reward mean: 4.4178 +num rollout transitions: 50000, reward mean: 4.4033 +num rollout transitions: 50000, reward mean: 4.4163 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4125 +num rollout transitions: 50000, reward mean: 4.4137 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4135 +num rollout transitions: 50000, reward mean: 4.3877 +num rollout transitions: 50000, reward mean: 4.4073 +num rollout transitions: 50000, reward mean: 4.4024 +num rollout transitions: 50000, reward mean: 4.3946 +num rollout transitions: 50000, reward mean: 4.4101 +num rollout transitions: 50000, reward mean: 4.3900 +num rollout transitions: 50000, reward mean: 4.3999 +num rollout transitions: 50000, reward mean: 4.4056 +num rollout transitions: 50000, reward mean: 4.4117 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4124 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.3990 +num rollout transitions: 50000, reward mean: 4.4184 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4192 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4077 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4162 +---------------------------------------------------------------------------------- +| alpha | 0.137 | +| eval/episode_length | 345 | +| eval/episode_length_std | 49 | +| eval/normalized_episode_reward | 30 | +| eval/normalized_episode_reward_std | 5.45 | +| loss/actor | -448 | +| loss/alpha | 0.00685 | +| loss/critic | 32.7 | +| penalty/lcb_mean | 1.2 | +| penalty/lcb_std | 1.85 | +| rollout_info/dynamics_penalty_mean | 0.174 | +| rollout_info/dynamics_penalty_std | 0.341 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.42 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4155 +num rollout transitions: 50000, reward mean: 4.4075 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4535 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4167 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4565 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4547 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4545 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4554 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4196 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4034 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4134 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4410 +---------------------------------------------------------------------------------- +| alpha | 0.149 | +| eval/episode_length | 564 | +| eval/episode_length_std | 245 | +| eval/normalized_episode_reward | 52.2 | +| eval/normalized_episode_reward_std | 24.4 | +| loss/actor | -444 | +| loss/alpha | 0.00265 | +| loss/critic | 35 | +| penalty/lcb_mean | 1.2 | +| penalty/lcb_std | 1.83 | +| rollout_info/dynamics_penalty_mean | 0.158 | +| rollout_info/dynamics_penalty_std | 0.328 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4138 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4531 +num rollout transitions: 50000, reward mean: 4.4541 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4644 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4528 +num rollout transitions: 50000, reward mean: 4.4595 +num rollout transitions: 50000, reward mean: 4.4643 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4587 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4507 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4182 +---------------------------------------------------------------------------------- +| alpha | 0.161 | +| eval/episode_length | 422 | +| eval/episode_length_std | 287 | +| eval/normalized_episode_reward | 37.7 | +| eval/normalized_episode_reward_std | 29.8 | +| loss/actor | -437 | +| loss/alpha | -0.00143 | +| loss/critic | 37.2 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.82 | +| rollout_info/dynamics_penalty_mean | 0.149 | +| rollout_info/dynamics_penalty_std | 0.322 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4562 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4648 +num rollout transitions: 50000, reward mean: 4.4563 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4608 +num rollout transitions: 50000, reward mean: 4.4623 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4585 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4107 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4542 +num rollout transitions: 50000, reward mean: 4.4593 +num rollout transitions: 50000, reward mean: 4.4503 +num rollout transitions: 50000, reward mean: 4.4584 +num rollout transitions: 50000, reward mean: 4.4680 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4558 +num rollout transitions: 50000, reward mean: 4.4554 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4605 +num rollout transitions: 50000, reward mean: 4.4526 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4516 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4558 +num rollout transitions: 50000, reward mean: 4.4632 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4629 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4500 +num rollout transitions: 50000, reward mean: 4.4384 +---------------------------------------------------------------------------------- +| alpha | 0.153 | +| eval/episode_length | 155 | +| eval/episode_length_std | 9.53 | +| eval/normalized_episode_reward | 8.68 | +| eval/normalized_episode_reward_std | 0.848 | +| loss/actor | -423 | +| loss/alpha | 0.00639 | +| loss/critic | 38.3 | +| penalty/lcb_mean | 1.28 | +| penalty/lcb_std | 1.83 | +| rollout_info/dynamics_penalty_mean | 0.141 | +| rollout_info/dynamics_penalty_std | 0.314 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.45 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4581 +num rollout transitions: 50000, reward mean: 4.4627 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4526 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4558 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4597 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4635 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4636 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4605 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4587 +num rollout transitions: 50000, reward mean: 4.4618 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4513 +num rollout transitions: 50000, reward mean: 4.4543 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4554 +num rollout transitions: 50000, reward mean: 4.4546 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4511 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4559 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4605 +num rollout transitions: 50000, reward mean: 4.4391 +---------------------------------------------------------------------------------- +| alpha | 0.17 | +| eval/episode_length | 957 | +| eval/episode_length_std | 108 | +| eval/normalized_episode_reward | 96.5 | +| eval/normalized_episode_reward_std | 11.9 | +| loss/actor | -407 | +| loss/alpha | 0.00526 | +| loss/critic | 44.3 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 1.97 | +| rollout_info/dynamics_penalty_mean | 0.13 | +| rollout_info/dynamics_penalty_std | 0.303 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.45 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4681 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4484 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4596 +num rollout transitions: 50000, reward mean: 4.4542 +num rollout transitions: 50000, reward mean: 4.4521 +num rollout transitions: 50000, reward mean: 4.4552 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4641 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4529 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4486 +num rollout transitions: 50000, reward mean: 4.4572 +num rollout transitions: 50000, reward mean: 4.4481 +num rollout transitions: 50000, reward mean: 4.4531 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4529 +num rollout transitions: 50000, reward mean: 4.4620 +num rollout transitions: 50000, reward mean: 4.4541 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4509 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4593 +num rollout transitions: 50000, reward mean: 4.4675 +num rollout transitions: 50000, reward mean: 4.4628 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4399 +---------------------------------------------------------------------------------- +| alpha | 0.18 | +| eval/episode_length | 734 | +| eval/episode_length_std | 285 | +| eval/normalized_episode_reward | 73.5 | +| eval/normalized_episode_reward_std | 30.9 | +| loss/actor | -403 | +| loss/alpha | 0.0068 | +| loss/critic | 48.3 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.07 | +| rollout_info/dynamics_penalty_mean | 0.129 | +| rollout_info/dynamics_penalty_std | 0.302 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.45 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4593 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4543 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4567 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4523 +num rollout transitions: 50000, reward mean: 4.4543 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4529 +num rollout transitions: 50000, reward mean: 4.4169 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4563 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4489 +num rollout transitions: 50000, reward mean: 4.4527 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4065 +num rollout transitions: 50000, reward mean: 4.4021 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4557 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4471 +---------------------------------------------------------------------------------- +| alpha | 0.177 | +| eval/episode_length | 999 | +| eval/episode_length_std | 10.8 | +| eval/normalized_episode_reward | 96.3 | +| eval/normalized_episode_reward_std | 1.87 | +| loss/actor | -397 | +| loss/alpha | -0.00459 | +| loss/critic | 49.3 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.13 | +| rollout_info/dynamics_penalty_mean | 0.126 | +| rollout_info/dynamics_penalty_std | 0.299 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4523 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4500 +num rollout transitions: 50000, reward mean: 4.4602 +num rollout transitions: 50000, reward mean: 4.4513 +num rollout transitions: 50000, reward mean: 4.4500 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4575 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4160 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4586 +num rollout transitions: 50000, reward mean: 4.4535 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.3996 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4539 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4574 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4255 +---------------------------------------------------------------------------------- +| alpha | 0.173 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0.696 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 1.04 | +| loss/actor | -395 | +| loss/alpha | 0.00145 | +| loss/critic | 50 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.16 | +| rollout_info/dynamics_penalty_mean | 0.123 | +| rollout_info/dynamics_penalty_std | 0.297 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4175 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4651 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4511 +num rollout transitions: 50000, reward mean: 4.4549 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4510 +num rollout transitions: 50000, reward mean: 4.4564 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4450 +----------------------------------------------------------------------------------- +| alpha | 0.168 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 0.589 | +| loss/actor | -392 | +| loss/alpha | -8.69e-05 | +| loss/critic | 48.9 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.14 | +| rollout_info/dynamics_penalty_mean | 0.122 | +| rollout_info/dynamics_penalty_std | 0.296 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 500000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4500 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4525 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4164 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4696 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4563 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4510 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4524 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4441 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4493 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4434 +---------------------------------------------------------------------------------- +| alpha | 0.162 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 0.98 | +| loss/actor | -396 | +| loss/alpha | -0.0102 | +| loss/critic | 47.9 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.15 | +| rollout_info/dynamics_penalty_mean | 0.123 | +| rollout_info/dynamics_penalty_std | 0.296 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4142 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4581 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4695 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4525 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4651 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4556 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4521 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4570 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4534 +---------------------------------------------------------------------------------- +| alpha | 0.15 | +| eval/episode_length | 117 | +| eval/episode_length_std | 20.3 | +| eval/normalized_episode_reward | 1.22 | +| eval/normalized_episode_reward_std | 0.464 | +| loss/actor | -398 | +| loss/alpha | -0.00537 | +| loss/critic | 47.1 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.14 | +| rollout_info/dynamics_penalty_mean | 0.123 | +| rollout_info/dynamics_penalty_std | 0.297 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4612 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4595 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4569 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4545 +num rollout transitions: 50000, reward mean: 4.4544 +num rollout transitions: 50000, reward mean: 4.4507 +num rollout transitions: 50000, reward mean: 4.4669 +num rollout transitions: 50000, reward mean: 4.4536 +num rollout transitions: 50000, reward mean: 4.4586 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4526 +num rollout transitions: 50000, reward mean: 4.4548 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4595 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4426 +---------------------------------------------------------------------------------- +| alpha | 0.158 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 0.407 | +| loss/actor | -394 | +| loss/alpha | -0.00339 | +| loss/critic | 49.7 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.23 | +| rollout_info/dynamics_penalty_mean | 0.124 | +| rollout_info/dynamics_penalty_std | 0.298 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4494 +num rollout transitions: 50000, reward mean: 4.4606 +num rollout transitions: 50000, reward mean: 4.4521 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4489 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4545 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4707 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4524 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4507 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4544 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4635 +num rollout transitions: 50000, reward mean: 4.4511 +num rollout transitions: 50000, reward mean: 4.4597 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4513 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4625 +num rollout transitions: 50000, reward mean: 4.4618 +num rollout transitions: 50000, reward mean: 4.4381 +---------------------------------------------------------------------------------- +| alpha | 0.148 | +| eval/episode_length | 996 | +| eval/episode_length_std | 26.4 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 3.21 | +| loss/actor | -393 | +| loss/alpha | 0.00278 | +| loss/critic | 49.3 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.23 | +| rollout_info/dynamics_penalty_mean | 0.12 | +| rollout_info/dynamics_penalty_std | 0.293 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4708 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4708 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4516 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4524 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4509 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4635 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4594 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4550 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4620 +num rollout transitions: 50000, reward mean: 4.4628 +num rollout transitions: 50000, reward mean: 4.4614 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4380 +---------------------------------------------------------------------------------- +| alpha | 0.152 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.06 | +| loss/actor | -395 | +| loss/alpha | 0.00469 | +| loss/critic | 50.5 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.27 | +| rollout_info/dynamics_penalty_mean | 0.119 | +| rollout_info/dynamics_penalty_std | 0.292 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4155 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4586 +num rollout transitions: 50000, reward mean: 4.4646 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4648 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4572 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4591 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4547 +num rollout transitions: 50000, reward mean: 4.4639 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4579 +num rollout transitions: 50000, reward mean: 4.4145 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4568 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4559 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4614 +num rollout transitions: 50000, reward mean: 4.4536 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4406 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 2.28 | +| loss/actor | -396 | +| loss/alpha | -0.00365 | +| loss/critic | 49.2 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 0.118 | +| rollout_info/dynamics_penalty_std | 0.291 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4667 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4519 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4484 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4748 +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4561 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4213 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4530 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4601 +num rollout transitions: 50000, reward mean: 4.4489 +num rollout transitions: 50000, reward mean: 4.4610 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4552 +num rollout transitions: 50000, reward mean: 4.4701 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4398 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 997 | +| eval/episode_length_std | 28 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 3.29 | +| loss/actor | -398 | +| loss/alpha | 0.00516 | +| loss/critic | 49.9 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 0.12 | +| rollout_info/dynamics_penalty_std | 0.294 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4575 +num rollout transitions: 50000, reward mean: 4.4568 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4552 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4543 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4526 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4504 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4529 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4494 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4654 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4212 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 1.59 | +| loss/actor | -400 | +| loss/alpha | -0.0116 | +| loss/critic | 48.2 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 0.125 | +| rollout_info/dynamics_penalty_std | 0.298 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4082 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4597 +num rollout transitions: 50000, reward mean: 4.4557 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4552 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4441 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4143 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4516 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4524 +num rollout transitions: 50000, reward mean: 4.4610 +num rollout transitions: 50000, reward mean: 4.4557 +num rollout transitions: 50000, reward mean: 4.4614 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4563 +num rollout transitions: 50000, reward mean: 4.4639 +num rollout transitions: 50000, reward mean: 4.4486 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.195 | +| loss/actor | -397 | +| loss/alpha | 0.0013 | +| loss/critic | 50.5 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 0.122 | +| rollout_info/dynamics_penalty_std | 0.296 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4546 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4517 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4568 +num rollout transitions: 50000, reward mean: 4.4551 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4552 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4666 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4372 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.765 | +| loss/actor | -398 | +| loss/alpha | 0.00238 | +| loss/critic | 48 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.2 | +| rollout_info/dynamics_penalty_mean | 0.128 | +| rollout_info/dynamics_penalty_std | 0.302 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4611 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4158 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4149 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4213 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4147 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4484 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4486 +num rollout transitions: 50000, reward mean: 4.4340 +---------------------------------------------------------------------------------- +| alpha | 0.136 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.762 | +| loss/actor | -399 | +| loss/alpha | -0.00198 | +| loss/critic | 47.9 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 0.123 | +| rollout_info/dynamics_penalty_std | 0.296 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4544 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4527 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4525 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4562 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4583 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4504 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4572 +num rollout transitions: 50000, reward mean: 4.4563 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4624 +num rollout transitions: 50000, reward mean: 4.4586 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4513 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4597 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4528 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4571 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4383 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 1.09 | +| loss/actor | -397 | +| loss/alpha | -0.00765 | +| loss/critic | 50.8 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 0.123 | +| rollout_info/dynamics_penalty_std | 0.296 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4573 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4554 +num rollout transitions: 50000, reward mean: 4.4578 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4604 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4511 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4156 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4567 +num rollout transitions: 50000, reward mean: 4.4413 +---------------------------------------------------------------------------------- +| alpha | 0.136 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.676 | +| loss/actor | -397 | +| loss/alpha | 0.011 | +| loss/critic | 47 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 0.12 | +| rollout_info/dynamics_penalty_std | 0.294 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4695 +num rollout transitions: 50000, reward mean: 4.4550 +num rollout transitions: 50000, reward mean: 4.4568 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4570 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4128 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4172 +num rollout transitions: 50000, reward mean: 4.4326 +----------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.44 | +| loss/actor | -400 | +| loss/alpha | -0.000477 | +| loss/critic | 50.1 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 0.126 | +| rollout_info/dynamics_penalty_std | 0.3 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1200000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4182 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4486 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4558 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4654 +num rollout transitions: 50000, reward mean: 4.4541 +num rollout transitions: 50000, reward mean: 4.4567 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4568 +num rollout transitions: 50000, reward mean: 4.4493 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4327 +---------------------------------------------------------------------------------- +| alpha | 0.142 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 1.19 | +| loss/actor | -398 | +| loss/alpha | -0.00508 | +| loss/critic | 52.5 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.127 | +| rollout_info/dynamics_penalty_std | 0.302 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4143 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4096 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4503 +num rollout transitions: 50000, reward mean: 4.4675 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4536 +num rollout transitions: 50000, reward mean: 4.4510 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4389 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.888 | +| loss/actor | -400 | +| loss/alpha | -0.0101 | +| loss/critic | 49.8 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.3 | +| rollout_info/dynamics_penalty_mean | 0.129 | +| rollout_info/dynamics_penalty_std | 0.304 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4168 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4565 +num rollout transitions: 50000, reward mean: 4.4513 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4489 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4051 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4307 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 0.744 | +| loss/actor | -404 | +| loss/alpha | 0.00786 | +| loss/critic | 50.6 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.129 | +| rollout_info/dynamics_penalty_std | 0.303 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4570 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4578 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4104 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4543 +num rollout transitions: 50000, reward mean: 4.4574 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4549 +num rollout transitions: 50000, reward mean: 4.4229 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 1.01 | +| loss/actor | -406 | +| loss/alpha | -0.00584 | +| loss/critic | 50.4 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.132 | +| rollout_info/dynamics_penalty_std | 0.307 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4192 +num rollout transitions: 50000, reward mean: 4.4099 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4045 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4192 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4181 +num rollout transitions: 50000, reward mean: 4.4146 +num rollout transitions: 50000, reward mean: 4.3994 +num rollout transitions: 50000, reward mean: 4.3980 +num rollout transitions: 50000, reward mean: 4.3972 +num rollout transitions: 50000, reward mean: 4.4095 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4162 +num rollout transitions: 50000, reward mean: 4.4099 +num rollout transitions: 50000, reward mean: 4.4119 +num rollout transitions: 50000, reward mean: 4.4135 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4529 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4523 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4582 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4124 +----------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.573 | +| loss/actor | -406 | +| loss/alpha | -1.96e-06 | +| loss/critic | 53.9 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0.135 | +| rollout_info/dynamics_penalty_std | 0.309 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1450000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4132 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4531 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4561 +num rollout transitions: 50000, reward mean: 4.4587 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4197 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4155 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4095 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4119 +num rollout transitions: 50000, reward mean: 4.4192 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4175 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4171 +num rollout transitions: 50000, reward mean: 4.4286 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 820 | +| eval/episode_length_std | 192 | +| eval/normalized_episode_reward | 84.5 | +| eval/normalized_episode_reward_std | 22.2 | +| loss/actor | -405 | +| loss/alpha | 0.00609 | +| loss/critic | 52.9 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0.136 | +| rollout_info/dynamics_penalty_std | 0.31 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4539 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4583 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4140 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4115 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4146 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4121 +num rollout transitions: 50000, reward mean: 4.4485 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.797 | +| loss/actor | -403 | +| loss/alpha | -0.00394 | +| loss/critic | 54.2 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0.133 | +| rollout_info/dynamics_penalty_std | 0.307 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4576 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4515 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4561 +num rollout transitions: 50000, reward mean: 4.4511 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4638 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4602 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4420 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.465 | +| loss/actor | -401 | +| loss/alpha | -0.00184 | +| loss/critic | 53.8 | +| penalty/lcb_mean | 1.43 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 0.13 | +| rollout_info/dynamics_penalty_std | 0.304 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4625 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4151 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4138 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4499 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4197 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 815 | +| eval/episode_length_std | 359 | +| eval/normalized_episode_reward | 81.8 | +| eval/normalized_episode_reward_std | 38.8 | +| loss/actor | -403 | +| loss/alpha | -0.00353 | +| loss/critic | 53.1 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 0.134 | +| rollout_info/dynamics_penalty_std | 0.308 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4062 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4155 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4168 +num rollout transitions: 50000, reward mean: 4.4170 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4133 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4041 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4437 +---------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 992 | +| eval/episode_length_std | 74.2 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 9.22 | +| loss/actor | -405 | +| loss/alpha | 0.00108 | +| loss/critic | 54.9 | +| penalty/lcb_mean | 1.45 | +| penalty/lcb_std | 2.43 | +| rollout_info/dynamics_penalty_mean | 0.141 | +| rollout_info/dynamics_penalty_std | 0.315 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4548 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4504 +num rollout transitions: 50000, reward mean: 4.4548 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4672 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4552 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4511 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4441 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4014 +num rollout transitions: 50000, reward mean: 4.4516 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4422 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.342 | +| loss/actor | -403 | +| loss/alpha | 0.00104 | +| loss/critic | 53.4 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 0.135 | +| rollout_info/dynamics_penalty_std | 0.308 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4490 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4486 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4553 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4486 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4499 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4154 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4556 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4259 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 997 | +| eval/episode_length_std | 34.2 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 4.36 | +| loss/actor | -403 | +| loss/alpha | 0.00411 | +| loss/critic | 52.7 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0.137 | +| rollout_info/dynamics_penalty_std | 0.311 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4213 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4516 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4166 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4454 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 2.85 | +| loss/actor | -404 | +| loss/alpha | -0.0148 | +| loss/critic | 51.9 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 0.136 | +| rollout_info/dynamics_penalty_std | 0.31 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4557 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4153 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4116 +num rollout transitions: 50000, reward mean: 4.4100 +num rollout transitions: 50000, reward mean: 4.4091 +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4108 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4144 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4617 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4680 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4391 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.282 | +| loss/actor | -404 | +| loss/alpha | 0.000349 | +| loss/critic | 55.2 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.48 | +| rollout_info/dynamics_penalty_mean | 0.135 | +| rollout_info/dynamics_penalty_std | 0.309 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4154 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4561 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4441 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4132 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4157 +num rollout transitions: 50000, reward mean: 4.4693 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4557 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4484 +num rollout transitions: 50000, reward mean: 4.4557 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4214 +---------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.185 | +| loss/actor | -404 | +| loss/alpha | 0.00664 | +| loss/critic | 53.8 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0.135 | +| rollout_info/dynamics_penalty_std | 0.309 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4143 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4513 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4499 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.3951 +num rollout transitions: 50000, reward mean: 4.4150 +num rollout transitions: 50000, reward mean: 4.4079 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4500 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4542 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4011 +num rollout transitions: 50000, reward mean: 4.4226 +---------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.828 | +| loss/actor | -405 | +| loss/alpha | -0.00655 | +| loss/critic | 56.2 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0.142 | +| rollout_info/dynamics_penalty_std | 0.316 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4561 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4139 +num rollout transitions: 50000, reward mean: 4.4173 +num rollout transitions: 50000, reward mean: 4.4122 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4164 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4066 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4531 +num rollout transitions: 50000, reward mean: 4.4380 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.451 | +| loss/actor | -404 | +| loss/alpha | 0.0101 | +| loss/critic | 58.9 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0.144 | +| rollout_info/dynamics_penalty_std | 0.319 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4137 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4109 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4199 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 995 | +| eval/episode_length_std | 41.9 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 4.86 | +| loss/actor | -406 | +| loss/alpha | -0.00581 | +| loss/critic | 56.6 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0.146 | +| rollout_info/dynamics_penalty_std | 0.321 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4166 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4107 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4169 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4066 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4199 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4143 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 987 | +| eval/episode_length_std | 65.1 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 8.22 | +| loss/actor | -407 | +| loss/alpha | 0.00463 | +| loss/critic | 54.5 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.44 | +| rollout_info/dynamics_penalty_mean | 0.143 | +| rollout_info/dynamics_penalty_std | 0.318 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4107 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4103 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4196 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4094 +num rollout transitions: 50000, reward mean: 4.4185 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.533 | +| loss/actor | -407 | +| loss/alpha | -0.00644 | +| loss/critic | 55.5 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0.145 | +| rollout_info/dynamics_penalty_std | 0.32 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4021 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.3948 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4146 +num rollout transitions: 50000, reward mean: 4.4061 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4172 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4519 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4096 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4164 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4142 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 2.02 | +| loss/actor | -406 | +| loss/alpha | -0.00416 | +| loss/critic | 57.3 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.53 | +| rollout_info/dynamics_penalty_mean | 0.147 | +| rollout_info/dynamics_penalty_std | 0.322 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4112 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4130 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4076 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4070 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4536 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4149 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.32 | +| loss/actor | -407 | +| loss/alpha | 0.00855 | +| loss/critic | 57 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0.149 | +| rollout_info/dynamics_penalty_std | 0.324 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4525 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4484 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4040 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4149 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4077 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4486 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4078 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4421 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.576 | +| loss/actor | -403 | +| loss/alpha | 0.0125 | +| loss/critic | 58.2 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.53 | +| rollout_info/dynamics_penalty_mean | 0.147 | +| rollout_info/dynamics_penalty_std | 0.323 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4527 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4162 +num rollout transitions: 50000, reward mean: 4.4163 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4127 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4544 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4115 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4143 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4499 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4406 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.13 | +| loss/actor | -400 | +| loss/alpha | -0.00874 | +| loss/critic | 61.7 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.66 | +| rollout_info/dynamics_penalty_mean | 0.141 | +| rollout_info/dynamics_penalty_std | 0.317 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4197 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4178 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4395 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.815 | +| loss/actor | -401 | +| loss/alpha | -0.00841 | +| loss/critic | 61.7 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.68 | +| rollout_info/dynamics_penalty_mean | 0.143 | +| rollout_info/dynamics_penalty_std | 0.318 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4119 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4500 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4084 +num rollout transitions: 50000, reward mean: 4.4077 +num rollout transitions: 50000, reward mean: 4.4084 +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4124 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4503 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4516 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4244 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.415 | +| loss/actor | -405 | +| loss/alpha | 0.00431 | +| loss/critic | 60 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.148 | +| rollout_info/dynamics_penalty_std | 0.323 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4138 +num rollout transitions: 50000, reward mean: 4.4161 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4156 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4517 +num rollout transitions: 50000, reward mean: 4.4486 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4185 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4172 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4007 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4161 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4326 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.651 | +| loss/actor | -405 | +| loss/alpha | 0.0116 | +| loss/critic | 61.6 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.68 | +| rollout_info/dynamics_penalty_mean | 0.146 | +| rollout_info/dynamics_penalty_std | 0.321 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4552 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4155 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4137 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4183 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4547 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4169 +num rollout transitions: 50000, reward mean: 4.4199 +---------------------------------------------------------------------------------- +| alpha | 0.113 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.669 | +| loss/actor | -403 | +| loss/alpha | -0.0162 | +| loss/critic | 59.3 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.147 | +| rollout_info/dynamics_penalty_std | 0.323 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4060 +num rollout transitions: 50000, reward mean: 4.4180 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4138 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4144 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4171 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4580 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4302 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.21 | +| loss/actor | -405 | +| loss/alpha | -0.00149 | +| loss/critic | 57.4 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.58 | +| rollout_info/dynamics_penalty_mean | 0.15 | +| rollout_info/dynamics_penalty_std | 0.325 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4107 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4130 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4169 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4170 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4112 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4183 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4535 +num rollout transitions: 50000, reward mean: 4.4596 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4213 +num rollout transitions: 50000, reward mean: 4.4342 +---------------------------------------------------------------------------------- +| alpha | 0.111 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.3 | +| loss/actor | -402 | +| loss/alpha | 0.0048 | +| loss/critic | 60 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.67 | +| rollout_info/dynamics_penalty_mean | 0.147 | +| rollout_info/dynamics_penalty_std | 0.323 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4183 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4221 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4105 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4347 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 119 | +| eval/normalized_episode_reward_std | 0.196 | +| loss/actor | -399 | +| loss/alpha | 0.000741 | +| loss/critic | 60.4 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.67 | +| rollout_info/dynamics_penalty_mean | 0.14 | +| rollout_info/dynamics_penalty_std | 0.316 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4545 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4175 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4536 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4516 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4312 +---------------------------------------------------------------------------------- +| alpha | 0.113 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 0.2 | +| loss/actor | -400 | +| loss/alpha | -0.0373 | +| loss/critic | 61.4 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.71 | +| rollout_info/dynamics_penalty_mean | 0.147 | +| rollout_info/dynamics_penalty_std | 0.322 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4103 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4132 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4129 +num rollout transitions: 50000, reward mean: 4.4137 +num rollout transitions: 50000, reward mean: 4.4173 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4178 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4274 +---------------------------------------------------------------------------------- +| alpha | 0.111 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 1.05 | +| loss/actor | -402 | +| loss/alpha | 0.041 | +| loss/critic | 60.8 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.66 | +| rollout_info/dynamics_penalty_mean | 0.149 | +| rollout_info/dynamics_penalty_std | 0.325 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4183 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4170 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4181 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4175 +num rollout transitions: 50000, reward mean: 4.4169 +num rollout transitions: 50000, reward mean: 4.4094 +num rollout transitions: 50000, reward mean: 4.4387 +---------------------------------------------------------------------------------- +| alpha | 0.109 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.501 | +| loss/actor | -403 | +| loss/alpha | -0.0575 | +| loss/critic | 60.8 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.68 | +| rollout_info/dynamics_penalty_mean | 0.149 | +| rollout_info/dynamics_penalty_std | 0.325 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4166 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4181 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4231 +---------------------------------------------------------------------------------- +| alpha | 0.101 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 120 | +| eval/normalized_episode_reward_std | 0.337 | +| loss/actor | -398 | +| loss/alpha | 0.0366 | +| loss/critic | 58.1 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0.149 | +| rollout_info/dynamics_penalty_std | 0.325 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4258 +---------------------------------------------------------------------------------- +| alpha | 0.115 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 120 | +| eval/normalized_episode_reward_std | 0.204 | +| loss/actor | -395 | +| loss/alpha | -0.0203 | +| loss/critic | 60.8 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.64 | +| rollout_info/dynamics_penalty_mean | 0.148 | +| rollout_info/dynamics_penalty_std | 0.325 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 128593.01s diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_123×tamp_26-0318-110814/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_123×tamp_26-0318-110814/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_123×tamp_26-0318-110814/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_123×tamp_26-0318-110814/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..48b0cad1bc36bdb9eacbeac67c512cd2304dd34f --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_123×tamp_26-0318-110814/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_realnvp.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 123, "device": "cuda:7", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_123×tamp_26-0112-030558/model", "reward_penalty_coef": 0.5, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/realnvp", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "realnvp", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "realnvp"} diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_123×tamp_26-0318-110814/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_123×tamp_26-0318-110814/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..d06535efa813727b49628e07de88e10a48c6509d --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_123×tamp_26-0318-110814/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.2562965846551931,99.95,4.109440351191388,4.505671187040756,0.21287187101662772,-278.6246375159073,-1.9307619682435417,18.482275273107835,0.8939949292231762,1.2071584224476115,0.31494215546244353,0.3756735185846512,50000.0,4.355048248864469,50000 +0.13677060235053493,345.07,49.02331180163168,30.02776946173322,5.447082265612611,-447.62471329833903,0.006849876750260615,32.73674646717071,1.2037359500360363,1.8547685876822342,0.17352986623925323,0.34126439181338386,50000.0,4.424630659816125,100000 +0.14919874622553436,564.49,245.3740611800685,52.222818964107944,24.448385082862142,-444.37318839477496,0.002653567175045657,35.008463403949996,1.202813702670341,1.8309557336759672,0.15783376131004398,0.32842655550476907,50000.0,4.435732303096179,150000 +0.16059332078754954,421.73,287.1303486223635,37.68324765437655,29.828621037984377,-436.79205233947624,-0.0014336136221883096,37.22415200681752,1.2735612232708935,1.8239683571541312,0.14935395181316838,0.3217422938071545,50000.0,4.440693942335108,200000 +0.15329229413449783,155.28,9.531085982195313,8.678706126486482,0.8478347700512227,-423.35239292541723,0.0063919766446575875,38.30236091873139,1.2797965982949817,1.8285023819541724,0.14102201537372397,0.3141904151545816,50000.0,4.4456751488004755,250000 +0.17016900944084204,956.87,108.0517149331745,96.46834644882496,11.919945540537851,-406.6038617938243,0.005259216194339232,44.27079335163114,1.3714752299165722,1.9744149254548573,0.12957606902678392,0.30282146230280177,50000.0,4.4460069574142675,300000 +0.17970624208539346,734.34,285.18356263992496,73.54734288251869,30.942198460695202,-402.53520986022966,0.006804775214158064,48.308081036987204,1.3994982654452364,2.071597890729855,0.1285539076111899,0.30228999381947175,50000.0,4.4456010978988925,350000 +0.17727195441395066,998.77,10.83407125691907,96.32847737148295,1.8654497462340431,-397.06309315612845,-0.004591769412234498,49.280573404693605,1.3714313888895642,2.131878442124147,0.1256137671852899,0.2988941421328732,50000.0,4.438661744171329,400000 +0.1734961306419947,999.93,0.6964912059746339,101.02615525954859,1.0419358503581504,-394.615610592653,0.0014546217159554166,50.04627300117473,1.3863261988568067,2.1624629747641473,0.12326339990900557,0.2972842142081863,50000.0,4.439689184636835,450000 +0.16839327440887675,1000.0,0.0,101.57146601260393,0.589431986215169,-392.17766181213614,-8.691082077095987e-05,48.89886192617434,1.368415412861094,2.1393240483379232,0.12244743199230493,0.29560388783339814,50000.0,4.439222676859381,500000 +0.1624662995088088,1000.0,0.0,107.36764921054788,0.9801258059485639,-395.711597117311,-0.01017028955569505,47.93916776727686,1.3487799158728184,2.1465677209889633,0.12347310452515549,0.29638607104624776,50000.0,4.43948145043345,550000 +0.15034671563714952,117.48,20.307870395489527,1.223350303852372,0.4642721063201892,-398.0838797790612,-0.005369780335463617,47.09351005447367,1.3427539820694798,2.144775352754612,0.12341569839532936,0.2967622040071479,50000.0,4.442520793985945,600000 +0.15771972882836977,1000.0,0.0,105.13124024291693,0.4072271485685792,-393.56090006774656,-0.003386539094932443,49.69021911956791,1.3908711564349943,2.2336954135048654,0.12419589070774605,0.29759290166912644,50000.0,4.441782766028803,650000 +0.14841130775153794,995.54,26.368321903374888,111.31674451079911,3.212391417078441,-393.22669093750795,0.0027815288406981857,49.29298169940911,1.3603382930433752,2.2339898620665375,0.11982901625862664,0.29292044759813535,50000.0,4.4449613822937835,700000 +0.15239748654663407,1000.0,0.0,113.61148902128342,1.0621376720366302,-394.6425856445203,0.00468516629274953,50.54631867053995,1.385942307010889,2.2695351572227485,0.11906935784953378,0.2923512021669127,50000.0,4.443589984427144,750000 +0.14536835721224547,1000.0,0.0,109.48655571294161,2.2758056485550346,-395.68213088927706,-0.0036510151282511593,49.240373845711254,1.3541743176436296,2.2447626256502082,0.11780293148092874,0.2911666057209853,50000.0,4.442930851582133,800000 +0.14093411753117857,997.19,27.95914698269602,110.69384925078876,3.289595119602984,-397.8977919598362,0.005156773979067889,49.85454020444917,1.3527333358967224,2.2811031608450376,0.1204897826303446,0.29365377723266733,50000.0,4.442920263981833,850000 +0.14028543112725,1000.0,0.0,112.26992213473027,1.59213530726404,-400.11499183044083,-0.011634441432096037,48.159766384621385,1.3353771037101665,2.2114541878390166,0.12462821235579505,0.2984027409962546,50000.0,4.440435674054458,900000 +0.14028446277111403,1000.0,0.0,113.52304408974607,0.1950654387672103,-396.9448829516616,0.0012983886731789374,50.460149383258994,1.3698563778007014,2.280089869499277,0.12195497824528095,0.2958880310700482,50000.0,4.442969466366179,950000 +0.13987203522354363,1000.0,0.0,115.42289821363924,0.7649267890292107,-398.36346318664465,0.002381851540654862,48.02841768312448,1.346389078855531,2.2033873968064985,0.12804913601539725,0.3018139278334012,50000.0,4.438487084178078,1000000 +0.1358215942320214,1000.0,0.0,113.21601771448186,0.7616734708372775,-399.123896049199,-0.001976873845830564,47.90091564044921,1.3433159326839215,2.2379849740541418,0.12265044584552719,0.29627313493164703,50000.0,4.436392188248865,1050000 +0.13962313138067722,1000.0,0.0,112.04141274433745,1.0949313768805817,-397.1284810498071,-0.007654424869939609,50.81926281810719,1.3795061200261043,2.2845186445188386,0.12315351663988222,0.2964418709100685,50000.0,4.444783711666735,1100000 +0.13644984276801086,1000.0,0.0,111.91288594391251,0.6757463926984825,-397.19769492675397,0.010966304549612178,46.951418127478725,1.3357320463311626,2.222335687088999,0.11990266909089026,0.29364198421081084,50000.0,4.440642294027108,1150000 +0.13536128084912782,1000.0,0.0,113.73718735054372,1.4362011196382578,-400.08603643799415,-0.0004772077859938343,50.09634022321661,1.381314369939577,2.3083530335854907,0.1260025478221595,0.3000688822051231,50000.0,4.438059430317821,1200000 +0.14200837622702017,1000.0,0.0,110.78221057831928,1.1931690377406308,-398.417891323855,-0.0050768706561625096,52.49481675874716,1.4149273454892464,2.352769737187642,0.12712643016549413,0.30176075230497434,50000.0,4.439223036544061,1250000 +0.13241982518166667,1000.0,0.0,109.94090220395454,0.8883019144700424,-400.1613285504182,-0.01011785453204062,49.83468976984045,1.3709961335766458,2.302878593075268,0.12940912697314316,0.303822857542384,50000.0,4.437992919685981,1300000 +0.13411146662160492,1000.0,0.0,107.34492104775808,0.7441065563478622,-403.72915704651047,0.007856403983197998,50.63899278242113,1.3980966449546939,2.3487233474302203,0.12938446810846596,0.30302647435890273,50000.0,4.437591791941398,1350000 +0.1321582050530639,1000.0,0.0,104.32380466723457,1.0066140154935916,-406.1880289880342,-0.005840242081247284,50.42958904552433,1.3700351817798588,2.352139056804159,0.132401835793771,0.3067398087669085,50000.0,4.436635640575044,1400000 +0.12829691251188588,1000.0,0.0,109.7937409891597,0.5726283862620435,-405.5359818603575,-1.9594220817236154e-06,53.85677512247119,1.4111484317410068,2.4175112856411687,0.1345275259891773,0.30891989058070424,50000.0,4.427094404502831,1450000 +0.12758720536857696,820.32,192.39541990390518,84.47188816181253,22.22702652044985,-404.8532072955358,0.006087217094078615,52.90500500890674,1.407843730260142,2.410388388317815,0.13642196939092127,0.31003254905747224,50000.0,4.4300272370378755,1500000 +0.12506262028351528,1000.0,0.0,113.03561376073212,0.7973883525655698,-402.6938392443827,-0.0039433038970083005,54.19412398004456,1.408528060688982,2.4142269829642196,0.13267886602684031,0.30688034892622285,50000.0,4.435482572758662,1550000 +0.1292184319993837,1000.0,0.0,113.78757234479855,0.4653530253816545,-401.3569497241123,-0.0018445184161140393,53.80967079072939,1.4288780136918877,2.3938804516041254,0.1299335821017033,0.3044093242590896,50000.0,4.438803738035661,1600000 +0.13300239733964245,814.91,359.14345587800983,81.81513693417527,38.842710330774594,-402.5057263116482,-0.003531445878409029,53.07621723705273,1.4177827658975113,2.3932342145038024,0.13358071939486624,0.307681519744752,50000.0,4.434430826871486,1650000 +0.12221515977531539,991.8,74.2464813981107,111.8886994777786,9.218876167125325,-405.17299745849874,0.0010789390869811425,54.861629383297206,1.446567682437874,2.4262941490948084,0.140583067805026,0.31468927271648534,50000.0,4.427517050185722,1700000 +0.12436337527215632,1000.0,0.0,109.97403195423455,0.3415323253988954,-402.9974547778359,0.0010443123466520763,53.43037100183434,1.4219535051941816,2.382640282257808,0.13475206000963605,0.3083364966313661,50000.0,4.439068475673535,1750000 +0.12642373963847714,996.56,34.22756783646772,111.31529992547023,4.362057961941356,-403.25356419494267,0.004106809866949921,52.744297574005145,1.4208846161270121,2.4129272806811155,0.13663181361336701,0.31099119491386806,50000.0,4.436300655936239,1800000 +0.12321870933145256,1000.0,0.0,112.7291980948363,2.853306169624558,-403.5824431396478,-0.0148273421111705,51.94154040832529,1.407254454181191,2.395574766321209,0.1355880463764871,0.30994621118341725,50000.0,4.4351930395476575,1850000 +0.12423011841043793,1000.0,0.0,110.17326877963333,0.2815248858677803,-404.0260316137658,0.0003487646799537102,55.15448864482939,1.4583961983573357,2.478884279478837,0.1348157879336729,0.308925853879315,50000.0,4.4351428354713045,1900000 +0.12160326638489777,1000.0,0.0,115.74890176121193,0.18492837098488166,-404.03278032776007,0.006643030590526675,53.84827208595215,1.4354789809548834,2.423108959009664,0.1348956092684489,0.30861222401135674,50000.0,4.436916720658598,1950000 +0.12154138157471973,1000.0,0.0,111.88477921184175,0.8280265908199217,-405.3899605499308,-0.006553917503692163,56.23770637504574,1.4581810707390297,2.487800487103463,0.14166325981759004,0.31565375974690096,50000.0,4.432376281985036,2000000 +0.12804971285760333,1000.0,0.0,111.80935966798596,0.4510718869450632,-403.9577013793965,0.010115321214198909,58.86716594591127,1.4988441546154123,2.5418226817130782,0.1443147975467708,0.31891024510089894,50000.0,4.432647444975275,2050000 +0.12550903643801867,994.75,41.91285602294361,108.91037479338921,4.8589222893613835,-406.1005949487353,-0.005810617416910866,56.57368013326619,1.4739675959145833,2.4868861530745154,0.1462069111858045,0.32081024402049085,50000.0,4.431443488439966,2100000 +0.12014880881518131,987.01,65.07142153049985,113.35413252099772,8.218090113528884,-407.23799872986075,0.004632700676322028,54.47855671878791,1.4402821670031414,2.4405147955823123,0.14300012184509872,0.3176716029776081,50000.0,4.430876596769625,2150000 +0.12105416575536035,1000.0,0.0,115.02391013457722,0.5333618072582679,-406.7516521752915,-0.006441149105541416,55.476866829662555,1.4575047085845663,2.4612410236096216,0.145007843957703,0.319789844180028,50000.0,4.431345519898931,2200000 +0.11928022605583011,1000.0,0.0,111.06003399990017,2.0208923990553447,-406.3086290429605,-0.004163322022780781,57.33317871112863,1.468699978631742,2.5279220663201976,0.1467115863532408,0.32211871561237726,50000.0,4.429129129093046,2250000 +0.12033215657413056,1000.0,0.0,116.89221034578354,0.3199213004697844,-407.34419332580404,0.00854745317878196,56.977701971053996,1.4748770961951978,2.5382961035895564,0.14853480444078104,0.3237424160619104,50000.0,4.431727305938403,2300000 +0.1262209791281828,1000.0,0.0,116.79660847996828,0.5763872344837995,-403.17758619750526,0.012480636033676714,58.17224503816549,1.5191442476987702,2.525466428370415,0.14651181437217298,0.32271261549845315,50000.0,4.433027130115833,2350000 +0.12951071813821885,1000.0,0.0,116.41875256319263,0.12960315864057406,-400.21832328125214,-0.008736645341627417,61.71408813346823,1.5680083651804924,2.6571224066901653,0.14149272496534754,0.31684999222534505,50000.0,4.4350499617761745,2400000 +0.12127063857391406,1000.0,0.0,116.51126828218747,0.8153856288697633,-401.22584747254047,-0.00840917219582953,61.737201543082776,1.529055884745097,2.6784123282527883,0.1431996248685192,0.318371416212369,50000.0,4.432786686938092,2450000 +0.11620186005383708,1000.0,0.0,116.05785291806939,0.415145290609115,-404.6862095867958,0.004311407960504318,60.03423215095564,1.4995226066553684,2.6257603562068383,0.14817223860293632,0.3229188115700115,50000.0,4.429303309031345,2500000 +0.1206485978215939,1000.0,0.0,113.96328036461728,0.6509233445295992,-405.07577575562044,0.011560581987015851,61.59371539110181,1.5264867098593566,2.6815906445228594,0.14566608703495382,0.32137853589934506,50000.0,4.4307324854667565,2550000 +0.11262242411032318,1000.0,0.0,115.18892446127258,0.6688377935353488,-403.20084520508107,-0.01615094819255165,59.25965220184329,1.4924329505205187,2.633140419341314,0.14700886274723196,0.3228742461445589,50000.0,4.429735622694175,2600000 +0.11361812325477623,1000.0,0.0,114.54745085921387,0.2101198980551512,-405.3058022741699,-0.0014872253645583909,57.382704967345774,1.4902452820599035,2.576799491569963,0.14951677450195694,0.3252684543333959,50000.0,4.431499533264198,2650000 +0.11110232398077752,1000.0,0.0,115.3893800094925,0.2996738421931553,-401.72380522155845,0.0048032628326489905,60.0490629597474,1.5180815610981084,2.6657971949911516,0.14744278827592663,0.32319802544945325,50000.0,4.430153426155921,2700000 +0.11411352562204087,1000.0,0.0,118.84744462276504,0.19557582810770074,-398.942320763552,0.0007412994718552031,60.35718610090216,1.5292396834218627,2.674054513003851,0.14021322467262348,0.31601856500207387,50000.0,4.431795015375817,2750000 +0.11322617896869867,1000.0,0.0,118.20277268418388,0.19951578053019858,-400.11109016296604,-0.03725056465368753,61.42888835613272,1.536030242044939,2.707579003703576,0.1467953649799683,0.32230829157726554,50000.0,4.4330153336342955,2800000 +0.11061313767299223,1000.0,0.0,118.13635994620586,1.0487032950674298,-402.24490866760163,0.04099736177548789,60.81698150590921,1.5320284834992652,2.663621263201236,0.149248924613536,0.3248168120413277,50000.0,4.425716877865095,2850000 +0.10923507563024824,1000.0,0.0,117.2715438685947,0.5012493925848521,-403.149183563233,-0.0574745873564107,60.8479871122742,1.5316776946926263,2.6816788187241296,0.14862269989660318,0.3245140547001007,50000.0,4.427930002219488,2900000 +0.10125037699773756,1000.0,0.0,119.89125756444083,0.33716545212610205,-398.25488590026555,0.03663875340446876,58.08015761081732,1.5092117326700563,2.609779624865062,0.14890443183422217,0.3254715311492588,50000.0,4.430132356990775,2950000 +0.11457310630440551,1000.0,0.0,119.69661817685974,0.20369297946821793,-394.53148138733167,-0.02026843654233938,60.75023724300426,1.5660993414926399,2.6426269601225965,0.148173578805223,0.3249951845074325,50000.0,4.428416281803233,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_42×tamp_26-0317-003138/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_42×tamp_26-0317-003138/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..3ed9e1bf2450c8b6e83f9b8c1e35e15944b920a1 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_42×tamp_26-0317-003138/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 3.9754 +num rollout transitions: 50000, reward mean: 4.1166 +num rollout transitions: 50000, reward mean: 4.1725 +num rollout transitions: 50000, reward mean: 4.1548 +num rollout transitions: 50000, reward mean: 4.1989 +num rollout transitions: 50000, reward mean: 4.1555 +num rollout transitions: 50000, reward mean: 4.1671 +num rollout transitions: 50000, reward mean: 4.2008 +num rollout transitions: 50000, reward mean: 4.2416 +num rollout transitions: 50000, reward mean: 4.2993 +num rollout transitions: 50000, reward mean: 4.3423 +num rollout transitions: 50000, reward mean: 4.3641 +num rollout transitions: 50000, reward mean: 4.3651 +num rollout transitions: 50000, reward mean: 4.3816 +num rollout transitions: 50000, reward mean: 4.3781 +num rollout transitions: 50000, reward mean: 4.3973 +num rollout transitions: 50000, reward mean: 4.3742 +num rollout transitions: 50000, reward mean: 4.3331 +num rollout transitions: 50000, reward mean: 4.3669 +num rollout transitions: 50000, reward mean: 4.3974 +num rollout transitions: 50000, reward mean: 4.3821 +num rollout transitions: 50000, reward mean: 4.3827 +num rollout transitions: 50000, reward mean: 4.3834 +num rollout transitions: 50000, reward mean: 4.3796 +num rollout transitions: 50000, reward mean: 4.3933 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4536 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4563 +num rollout transitions: 50000, reward mean: 4.4535 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4536 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4500 +num rollout transitions: 50000, reward mean: 4.4546 +num rollout transitions: 50000, reward mean: 4.4138 +---------------------------------------------------------------------------------- +| alpha | 0.266 | +| eval/episode_length | 157 | +| eval/episode_length_std | 58.4 | +| eval/normalized_episode_reward | 4.54 | +| eval/normalized_episode_reward_std | 4.99 | +| loss/actor | -277 | +| loss/alpha | -1.76 | +| loss/critic | 21.3 | +| penalty/lcb_mean | 0.997 | +| penalty/lcb_std | 1.26 | +| rollout_info/dynamics_penalty_mean | 0.31 | +| rollout_info/dynamics_penalty_std | 0.368 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.37 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4133 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4129 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4182 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4072 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4043 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4046 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4199 +num rollout transitions: 50000, reward mean: 4.4129 +num rollout transitions: 50000, reward mean: 4.4181 +num rollout transitions: 50000, reward mean: 4.4254 +---------------------------------------------------------------------------------- +| alpha | 0.153 | +| eval/episode_length | 40 | +| eval/episode_length_std | 0.399 | +| eval/normalized_episode_reward | 0.532 | +| eval/normalized_episode_reward_std | -0.0204 | +| loss/actor | -425 | +| loss/alpha | 0.0101 | +| loss/critic | 38.1 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 1.89 | +| rollout_info/dynamics_penalty_mean | 0.151 | +| rollout_info/dynamics_penalty_std | 0.322 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4168 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4173 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4161 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4525 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4148 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4127 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4481 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4484 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4131 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4160 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4146 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4401 +---------------------------------------------------------------------------------- +| alpha | 0.149 | +| eval/episode_length | 174 | +| eval/episode_length_std | 107 | +| eval/normalized_episode_reward | 9.35 | +| eval/normalized_episode_reward_std | 11.2 | +| loss/actor | -425 | +| loss/alpha | -0.0133 | +| loss/critic | 37.8 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 1.86 | +| rollout_info/dynamics_penalty_mean | 0.157 | +| rollout_info/dynamics_penalty_std | 0.329 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4032 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4182 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4132 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4568 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4531 +num rollout transitions: 50000, reward mean: 4.4653 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4617 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4176 +---------------------------------------------------------------------------------- +| alpha | 0.147 | +| eval/episode_length | 430 | +| eval/episode_length_std | 182 | +| eval/normalized_episode_reward | 38.7 | +| eval/normalized_episode_reward_std | 19.9 | +| loss/actor | -418 | +| loss/alpha | 0.0132 | +| loss/critic | 35.8 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.79 | +| rollout_info/dynamics_penalty_mean | 0.15 | +| rollout_info/dynamics_penalty_std | 0.323 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4568 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4504 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4585 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4708 +num rollout transitions: 50000, reward mean: 4.4634 +num rollout transitions: 50000, reward mean: 4.4511 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4168 +num rollout transitions: 50000, reward mean: 4.4162 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4489 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4182 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4196 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4196 +num rollout transitions: 50000, reward mean: 4.4247 +---------------------------------------------------------------------------------- +| alpha | 0.166 | +| eval/episode_length | 571 | +| eval/episode_length_std | 299 | +| eval/normalized_episode_reward | 52.9 | +| eval/normalized_episode_reward_std | 32.6 | +| loss/actor | -396 | +| loss/alpha | 0.00373 | +| loss/critic | 37.2 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 1.78 | +| rollout_info/dynamics_penalty_mean | 0.146 | +| rollout_info/dynamics_penalty_std | 0.321 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4168 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4548 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4658 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4134 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4196 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4223 +---------------------------------------------------------------------------------- +| alpha | 0.17 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 95.7 | +| eval/normalized_episode_reward_std | 0.981 | +| loss/actor | -379 | +| loss/alpha | 0.00698 | +| loss/critic | 42.7 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 1.91 | +| rollout_info/dynamics_penalty_mean | 0.14 | +| rollout_info/dynamics_penalty_std | 0.315 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4513 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4627 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4559 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4493 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4620 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4366 +---------------------------------------------------------------------------------- +| alpha | 0.171 | +| eval/episode_length | 876 | +| eval/episode_length_std | 192 | +| eval/normalized_episode_reward | 89.7 | +| eval/normalized_episode_reward_std | 21.2 | +| loss/actor | -373 | +| loss/alpha | -0.00865 | +| loss/critic | 44.9 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 1.99 | +| rollout_info/dynamics_penalty_mean | 0.136 | +| rollout_info/dynamics_penalty_std | 0.31 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4500 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4504 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4586 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4536 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4149 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4045 +num rollout transitions: 50000, reward mean: 4.4286 +---------------------------------------------------------------------------------- +| alpha | 0.168 | +| eval/episode_length | 965 | +| eval/episode_length_std | 99.8 | +| eval/normalized_episode_reward | 100 | +| eval/normalized_episode_reward_std | 11.3 | +| loss/actor | -373 | +| loss/alpha | 0.00181 | +| loss/critic | 47.6 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.07 | +| rollout_info/dynamics_penalty_mean | 0.131 | +| rollout_info/dynamics_penalty_std | 0.305 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4489 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4582 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4696 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4154 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4158 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4493 +num rollout transitions: 50000, reward mean: 4.4183 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4484 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4616 +num rollout transitions: 50000, reward mean: 4.4539 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4334 +---------------------------------------------------------------------------------- +| alpha | 0.162 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.3 | +| eval/normalized_episode_reward_std | 1.87 | +| loss/actor | -372 | +| loss/alpha | -0.00328 | +| loss/critic | 49.2 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.17 | +| rollout_info/dynamics_penalty_mean | 0.128 | +| rollout_info/dynamics_penalty_std | 0.301 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4538 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4085 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4530 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4168 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4654 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4489 +num rollout transitions: 50000, reward mean: 4.4213 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4500 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4591 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4549 +---------------------------------------------------------------------------------- +| alpha | 0.168 | +| eval/episode_length | 996 | +| eval/episode_length_std | 28.1 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 4.04 | +| loss/actor | -369 | +| loss/alpha | 0.00679 | +| loss/critic | 51.5 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.18 | +| rollout_info/dynamics_penalty_mean | 0.121 | +| rollout_info/dynamics_penalty_std | 0.293 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4550 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4558 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4531 +num rollout transitions: 50000, reward mean: 4.4548 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4617 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4610 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4105 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4600 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4524 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4339 +---------------------------------------------------------------------------------- +| alpha | 0.166 | +| eval/episode_length | 996 | +| eval/episode_length_std | 30.5 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 4.04 | +| loss/actor | -370 | +| loss/alpha | -0.00527 | +| loss/critic | 51.1 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.17 | +| rollout_info/dynamics_penalty_mean | 0.12 | +| rollout_info/dynamics_penalty_std | 0.293 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4097 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4516 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4146 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4598 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4544 +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4503 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4528 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4367 +---------------------------------------------------------------------------------- +| alpha | 0.159 | +| eval/episode_length | 995 | +| eval/episode_length_std | 43.8 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 5.32 | +| loss/actor | -373 | +| loss/alpha | -0.00178 | +| loss/critic | 51.7 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 0.122 | +| rollout_info/dynamics_penalty_std | 0.294 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4093 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4542 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4571 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4182 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4551 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4598 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4503 +num rollout transitions: 50000, reward mean: 4.4081 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4331 +---------------------------------------------------------------------------------- +| alpha | 0.157 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 1.69 | +| loss/actor | -371 | +| loss/alpha | -0.00584 | +| loss/critic | 52 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 0.122 | +| rollout_info/dynamics_penalty_std | 0.294 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4525 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4167 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4133 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4693 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4146 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4133 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4634 +num rollout transitions: 50000, reward mean: 4.4572 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4524 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4597 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4650 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4660 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4355 +---------------------------------------------------------------------------------- +| alpha | 0.148 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 0.559 | +| loss/actor | -370 | +| loss/alpha | -0.00733 | +| loss/critic | 49.8 | +| penalty/lcb_mean | 1.45 | +| penalty/lcb_std | 2.19 | +| rollout_info/dynamics_penalty_mean | 0.123 | +| rollout_info/dynamics_penalty_std | 0.296 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4585 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4648 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4558 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4176 +num rollout transitions: 50000, reward mean: 4.4102 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4577 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4519 +num rollout transitions: 50000, reward mean: 4.4553 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4701 +num rollout transitions: 50000, reward mean: 4.4497 +---------------------------------------------------------------------------------- +| alpha | 0.151 | +| eval/episode_length | 993 | +| eval/episode_length_std | 36.7 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 4.14 | +| loss/actor | -369 | +| loss/alpha | 0.0058 | +| loss/critic | 50.2 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 0.122 | +| rollout_info/dynamics_penalty_std | 0.294 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4166 +num rollout transitions: 50000, reward mean: 4.4503 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4125 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4531 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4603 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4579 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4504 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4610 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4377 +---------------------------------------------------------------------------------- +| alpha | 0.149 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 0.374 | +| loss/actor | -371 | +| loss/alpha | 0.000226 | +| loss/critic | 51 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 0.122 | +| rollout_info/dynamics_penalty_std | 0.295 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4612 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4663 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4553 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4531 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4494 +num rollout transitions: 50000, reward mean: 4.4549 +num rollout transitions: 50000, reward mean: 4.4573 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4509 +num rollout transitions: 50000, reward mean: 4.4525 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4163 +num rollout transitions: 50000, reward mean: 4.4558 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4271 +---------------------------------------------------------------------------------- +| alpha | 0.148 | +| eval/episode_length | 999 | +| eval/episode_length_std | 14.8 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.9 | +| loss/actor | -370 | +| loss/alpha | 0.000421 | +| loss/critic | 52 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 0.118 | +| rollout_info/dynamics_penalty_std | 0.291 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4172 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4133 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4074 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4199 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4592 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4499 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4141 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4532 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4214 +---------------------------------------------------------------------------------- +| alpha | 0.152 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.902 | +| loss/actor | -370 | +| loss/alpha | -0.00462 | +| loss/critic | 51.6 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.25 | +| rollout_info/dynamics_penalty_mean | 0.128 | +| rollout_info/dynamics_penalty_std | 0.302 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4056 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4119 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4150 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4123 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4130 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4158 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4044 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4065 +num rollout transitions: 50000, reward mean: 4.4104 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4469 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 992 | +| eval/episode_length_std | 28.8 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 3.58 | +| loss/actor | -373 | +| loss/alpha | 0.0144 | +| loss/critic | 51.3 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.29 | +| rollout_info/dynamics_penalty_mean | 0.133 | +| rollout_info/dynamics_penalty_std | 0.307 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4594 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4128 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4083 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4162 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4097 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4599 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4553 +num rollout transitions: 50000, reward mean: 4.4141 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4147 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4107 +num rollout transitions: 50000, reward mean: 4.4105 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4135 +num rollout transitions: 50000, reward mean: 4.4156 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4563 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4283 +---------------------------------------------------------------------------------- +| alpha | 0.151 | +| eval/episode_length | 961 | +| eval/episode_length_std | 105 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 13 | +| loss/actor | -372 | +| loss/alpha | -0.0105 | +| loss/critic | 54 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 0.131 | +| rollout_info/dynamics_penalty_std | 0.305 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4185 +num rollout transitions: 50000, reward mean: 4.3961 +num rollout transitions: 50000, reward mean: 4.4486 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4150 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4112 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4490 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4576 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4110 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4270 +----------------------------------------------------------------------------------- +| alpha | 0.144 | +| eval/episode_length | 990 | +| eval/episode_length_std | 36.8 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 4.62 | +| loss/actor | -372 | +| loss/alpha | -1.32e-05 | +| loss/critic | 51.4 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.27 | +| rollout_info/dynamics_penalty_mean | 0.126 | +| rollout_info/dynamics_penalty_std | 0.299 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1050000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4199 +num rollout transitions: 50000, reward mean: 4.4181 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4489 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.3893 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4135 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4144 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4183 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4192 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4242 +----------------------------------------------------------------------------------- +| alpha | 0.136 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.652 | +| loss/actor | -376 | +| loss/alpha | -2.82e-05 | +| loss/critic | 51 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.29 | +| rollout_info/dynamics_penalty_mean | 0.132 | +| rollout_info/dynamics_penalty_std | 0.306 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1100000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4170 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4181 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.3997 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4170 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4173 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4184 +num rollout transitions: 50000, reward mean: 4.4240 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.635 | +| loss/actor | -374 | +| loss/alpha | -0.00169 | +| loss/critic | 53.8 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.131 | +| rollout_info/dynamics_penalty_std | 0.305 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4507 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4114 +num rollout transitions: 50000, reward mean: 4.4199 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4230 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4499 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4044 +num rollout transitions: 50000, reward mean: 4.4065 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4108 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4552 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4354 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 964 | +| eval/episode_length_std | 105 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 12.6 | +| loss/actor | -372 | +| loss/alpha | 0.00445 | +| loss/critic | 53 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 0.13 | +| rollout_info/dynamics_penalty_std | 0.304 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4109 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4185 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4080 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4178 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4127 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4557 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4627 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4519 +num rollout transitions: 50000, reward mean: 4.4068 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4426 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 978 | +| eval/episode_length_std | 78.4 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 9.84 | +| loss/actor | -370 | +| loss/alpha | -0.00322 | +| loss/critic | 54.1 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 0.131 | +| rollout_info/dynamics_penalty_std | 0.305 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4134 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4172 +num rollout transitions: 50000, reward mean: 4.4151 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4011 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4130 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4135 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4178 +num rollout transitions: 50000, reward mean: 4.4109 +num rollout transitions: 50000, reward mean: 4.4117 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4108 +num rollout transitions: 50000, reward mean: 4.4192 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4014 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4241 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 999 | +| eval/episode_length_std | 5.87 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 1.82 | +| loss/actor | -373 | +| loss/alpha | 0.000242 | +| loss/critic | 53.5 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 0.14 | +| rollout_info/dynamics_penalty_std | 0.313 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4175 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4543 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4151 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4172 +num rollout transitions: 50000, reward mean: 4.4004 +num rollout transitions: 50000, reward mean: 4.4078 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4622 +num rollout transitions: 50000, reward mean: 4.4569 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4588 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4180 +num rollout transitions: 50000, reward mean: 4.4162 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4129 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4108 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4301 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 910 | +| eval/episode_length_std | 151 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 18.4 | +| loss/actor | -375 | +| loss/alpha | -0.00137 | +| loss/critic | 56 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 0.134 | +| rollout_info/dynamics_penalty_std | 0.308 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.3975 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4070 +num rollout transitions: 50000, reward mean: 4.4107 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4068 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4185 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4064 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4120 +num rollout transitions: 50000, reward mean: 4.4160 +num rollout transitions: 50000, reward mean: 4.3932 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4072 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4102 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4123 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4142 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4081 +num rollout transitions: 50000, reward mean: 4.4032 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4117 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 847 | +| eval/episode_length_std | 180 | +| eval/normalized_episode_reward | 95.5 | +| eval/normalized_episode_reward_std | 22.4 | +| loss/actor | -377 | +| loss/alpha | 0.000404 | +| loss/critic | 59.7 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0.143 | +| rollout_info/dynamics_penalty_std | 0.318 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.42 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4116 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4146 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4050 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4197 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4171 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4125 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.3987 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4196 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4499 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4154 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4053 +num rollout transitions: 50000, reward mean: 4.4025 +num rollout transitions: 50000, reward mean: 4.4051 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4394 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.754 | +| loss/actor | -376 | +| loss/alpha | -0.00404 | +| loss/critic | 58.4 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0.14 | +| rollout_info/dynamics_penalty_std | 0.314 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4072 +num rollout transitions: 50000, reward mean: 4.4171 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4175 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4124 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4230 +num rollout transitions: 50000, reward mean: 4.4168 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4161 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4561 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4484 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4593 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4637 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4299 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 993 | +| eval/episode_length_std | 39.1 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 4.98 | +| loss/actor | -379 | +| loss/alpha | 0.00115 | +| loss/critic | 56.3 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0.133 | +| rollout_info/dynamics_penalty_std | 0.307 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4576 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4548 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4146 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4119 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4184 +---------------------------------------------------------------------------------- +| alpha | 0.137 | +| eval/episode_length | 996 | +| eval/episode_length_std | 29.1 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 3.5 | +| loss/actor | -376 | +| loss/alpha | -0.00502 | +| loss/critic | 56.5 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0.136 | +| rollout_info/dynamics_penalty_std | 0.309 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4134 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4101 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4121 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4085 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4199 +num rollout transitions: 50000, reward mean: 4.4169 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4082 +num rollout transitions: 50000, reward mean: 4.4039 +num rollout transitions: 50000, reward mean: 4.4139 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4181 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4301 +----------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 536 | +| eval/episode_length_std | 221 | +| eval/normalized_episode_reward | 53.6 | +| eval/normalized_episode_reward_std | 26 | +| loss/actor | -373 | +| loss/alpha | -0.000448 | +| loss/critic | 54 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.36 | +| rollout_info/dynamics_penalty_mean | 0.144 | +| rollout_info/dynamics_penalty_std | 0.318 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1600000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4024 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4052 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4148 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4125 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4197 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4084 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4141 +num rollout transitions: 50000, reward mean: 4.4161 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.3937 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4126 +num rollout transitions: 50000, reward mean: 4.4083 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4105 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4145 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4437 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 732 | +| eval/episode_length_std | 172 | +| eval/normalized_episode_reward | 79.5 | +| eval/normalized_episode_reward_std | 20 | +| loss/actor | -374 | +| loss/alpha | 0.00614 | +| loss/critic | 55.9 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 0.147 | +| rollout_info/dynamics_penalty_std | 0.32 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4523 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4154 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4558 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4196 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4127 +num rollout transitions: 50000, reward mean: 4.4150 +num rollout transitions: 50000, reward mean: 4.4057 +num rollout transitions: 50000, reward mean: 4.4158 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4161 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4184 +num rollout transitions: 50000, reward mean: 4.4213 +num rollout transitions: 50000, reward mean: 4.4128 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4235 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 941 | +| eval/episode_length_std | 133 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 15.7 | +| loss/actor | -373 | +| loss/alpha | 0.00312 | +| loss/critic | 57.2 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0.14 | +| rollout_info/dynamics_penalty_std | 0.315 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4481 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4538 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4040 +num rollout transitions: 50000, reward mean: 4.4163 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4049 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4155 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4098 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4090 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4122 +num rollout transitions: 50000, reward mean: 4.4148 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4162 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.97 | +| loss/actor | -376 | +| loss/alpha | -0.00886 | +| loss/critic | 56.7 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0.147 | +| rollout_info/dynamics_penalty_std | 0.321 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4171 +num rollout transitions: 50000, reward mean: 4.4060 +num rollout transitions: 50000, reward mean: 4.4172 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4197 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4063 +num rollout transitions: 50000, reward mean: 4.4163 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.3930 +num rollout transitions: 50000, reward mean: 4.4173 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4163 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4056 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4127 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4160 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4151 +num rollout transitions: 50000, reward mean: 4.4308 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 990 | +| eval/episode_length_std | 47.3 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 6.23 | +| loss/actor | -376 | +| loss/alpha | -0.00714 | +| loss/critic | 57.7 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.48 | +| rollout_info/dynamics_penalty_mean | 0.15 | +| rollout_info/dynamics_penalty_std | 0.324 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.42 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4157 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4027 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4116 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4092 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4119 +num rollout transitions: 50000, reward mean: 4.4051 +num rollout transitions: 50000, reward mean: 4.3990 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4178 +num rollout transitions: 50000, reward mean: 4.4161 +num rollout transitions: 50000, reward mean: 4.4133 +num rollout transitions: 50000, reward mean: 4.4109 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 738 | +| eval/episode_length_std | 221 | +| eval/normalized_episode_reward | 81.5 | +| eval/normalized_episode_reward_std | 26.8 | +| loss/actor | -374 | +| loss/alpha | 0.00625 | +| loss/critic | 59.7 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.5 | +| rollout_info/dynamics_penalty_mean | 0.148 | +| rollout_info/dynamics_penalty_std | 0.323 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4036 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4033 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4081 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4134 +num rollout transitions: 50000, reward mean: 4.4164 +num rollout transitions: 50000, reward mean: 4.4161 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4111 +num rollout transitions: 50000, reward mean: 4.4101 +num rollout transitions: 50000, reward mean: 4.4128 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4324 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 865 | +| eval/episode_length_std | 171 | +| eval/normalized_episode_reward | 98.4 | +| eval/normalized_episode_reward_std | 21.2 | +| loss/actor | -374 | +| loss/alpha | 0.00741 | +| loss/critic | 60.4 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.48 | +| rollout_info/dynamics_penalty_mean | 0.146 | +| rollout_info/dynamics_penalty_std | 0.321 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.42 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4016 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.3830 +num rollout transitions: 50000, reward mean: 4.4142 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4029 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4185 +num rollout transitions: 50000, reward mean: 4.4197 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4137 +num rollout transitions: 50000, reward mean: 4.4170 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4316 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 997 | +| eval/episode_length_std | 18.6 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 2.43 | +| loss/actor | -378 | +| loss/alpha | -0.0062 | +| loss/critic | 59.8 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.53 | +| rollout_info/dynamics_penalty_mean | 0.141 | +| rollout_info/dynamics_penalty_std | 0.315 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4032 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4108 +num rollout transitions: 50000, reward mean: 4.4119 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4118 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4131 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4185 +num rollout transitions: 50000, reward mean: 4.4170 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4175 +num rollout transitions: 50000, reward mean: 4.4220 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 997 | +| eval/episode_length_std | 31.5 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 3.96 | +| loss/actor | -374 | +| loss/alpha | 0.00082 | +| loss/critic | 62.6 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0.139 | +| rollout_info/dynamics_penalty_std | 0.313 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4178 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4113 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4182 +num rollout transitions: 50000, reward mean: 4.4185 +num rollout transitions: 50000, reward mean: 4.4155 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4534 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4150 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4571 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4164 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4162 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 995 | +| eval/episode_length_std | 53.4 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 6.61 | +| loss/actor | -373 | +| loss/alpha | -0.00523 | +| loss/critic | 60.8 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.55 | +| rollout_info/dynamics_penalty_mean | 0.138 | +| rollout_info/dynamics_penalty_std | 0.312 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4099 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4016 +num rollout transitions: 50000, reward mean: 4.4173 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4138 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4493 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4167 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4151 +num rollout transitions: 50000, reward mean: 4.4162 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4320 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 885 | +| eval/episode_length_std | 164 | +| eval/normalized_episode_reward | 99.3 | +| eval/normalized_episode_reward_std | 19.8 | +| loss/actor | -373 | +| loss/alpha | -0.00828 | +| loss/critic | 59.5 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.51 | +| rollout_info/dynamics_penalty_mean | 0.146 | +| rollout_info/dynamics_penalty_std | 0.319 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4122 +num rollout transitions: 50000, reward mean: 4.4062 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4178 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4537 +num rollout transitions: 50000, reward mean: 4.4450 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 984 | +| eval/episode_length_std | 72.1 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 9.44 | +| loss/actor | -372 | +| loss/alpha | 0.0144 | +| loss/critic | 60.8 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0.14 | +| rollout_info/dynamics_penalty_std | 0.313 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4504 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4135 +num rollout transitions: 50000, reward mean: 4.4114 +num rollout transitions: 50000, reward mean: 4.4037 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4149 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4014 +num rollout transitions: 50000, reward mean: 4.4048 +num rollout transitions: 50000, reward mean: 4.4173 +num rollout transitions: 50000, reward mean: 4.4091 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4176 +---------------------------------------------------------------------------------- +| alpha | 0.136 | +| eval/episode_length | 955 | +| eval/episode_length_std | 109 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 13.4 | +| loss/actor | -368 | +| loss/alpha | -0.00928 | +| loss/critic | 58.7 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0.134 | +| rollout_info/dynamics_penalty_std | 0.307 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4009 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4126 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4185 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4134 +num rollout transitions: 50000, reward mean: 4.4121 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4571 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4104 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4079 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4253 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.6 | +| loss/actor | -370 | +| loss/alpha | 0.00253 | +| loss/critic | 58.7 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.52 | +| rollout_info/dynamics_penalty_mean | 0.139 | +| rollout_info/dynamics_penalty_std | 0.313 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4168 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4138 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4097 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4135 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4168 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4197 +num rollout transitions: 50000, reward mean: 4.4092 +num rollout transitions: 50000, reward mean: 4.4098 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4183 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4137 +num rollout transitions: 50000, reward mean: 4.4118 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4568 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4301 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 622 | +| eval/episode_length_std | 279 | +| eval/normalized_episode_reward | 67.4 | +| eval/normalized_episode_reward_std | 35.3 | +| loss/actor | -368 | +| loss/alpha | 0.00339 | +| loss/critic | 62.9 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.62 | +| rollout_info/dynamics_penalty_mean | 0.133 | +| rollout_info/dynamics_penalty_std | 0.307 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4441 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4481 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4093 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4308 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 955 | +| eval/episode_length_std | 104 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 12.9 | +| loss/actor | -362 | +| loss/alpha | 0.00651 | +| loss/critic | 62.9 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0.132 | +| rollout_info/dynamics_penalty_std | 0.306 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4184 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4111 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4067 +num rollout transitions: 50000, reward mean: 4.4112 +num rollout transitions: 50000, reward mean: 4.4167 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4301 +----------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 979 | +| eval/episode_length_std | 93.1 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 11.5 | +| loss/actor | -363 | +| loss/alpha | -0.000443 | +| loss/critic | 62.8 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0.132 | +| rollout_info/dynamics_penalty_std | 0.305 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2400000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4150 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4197 +num rollout transitions: 50000, reward mean: 4.4153 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4183 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4500 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4630 +num rollout transitions: 50000, reward mean: 4.4543 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4273 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.353 | +| loss/actor | -367 | +| loss/alpha | -0.0111 | +| loss/critic | 61.9 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.6 | +| rollout_info/dynamics_penalty_mean | 0.13 | +| rollout_info/dynamics_penalty_std | 0.304 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4557 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4527 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4157 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4167 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4184 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4168 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 978 | +| eval/episode_length_std | 86.8 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 10.9 | +| loss/actor | -366 | +| loss/alpha | 0.00844 | +| loss/critic | 61.7 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.58 | +| rollout_info/dynamics_penalty_mean | 0.131 | +| rollout_info/dynamics_penalty_std | 0.305 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4157 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4138 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4132 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4340 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.508 | +| loss/actor | -371 | +| loss/alpha | -0.00768 | +| loss/critic | 63 | +| penalty/lcb_mean | 1.69 | +| penalty/lcb_std | 2.6 | +| rollout_info/dynamics_penalty_mean | 0.134 | +| rollout_info/dynamics_penalty_std | 0.309 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4547 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4164 +num rollout transitions: 50000, reward mean: 4.4154 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4542 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4151 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4252 +---------------------------------------------------------------------------------- +| alpha | 0.137 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.742 | +| loss/actor | -367 | +| loss/alpha | 0.00184 | +| loss/critic | 60.9 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0.133 | +| rollout_info/dynamics_penalty_std | 0.308 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4493 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4503 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4499 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4572 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4172 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4505 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 0.919 | +| loss/actor | -365 | +| loss/alpha | 0.00282 | +| loss/critic | 58.7 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0.131 | +| rollout_info/dynamics_penalty_std | 0.306 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4178 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4262 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.348 | +| loss/actor | -369 | +| loss/alpha | -0.00416 | +| loss/critic | 59.7 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.53 | +| rollout_info/dynamics_penalty_mean | 0.135 | +| rollout_info/dynamics_penalty_std | 0.309 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4135 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4136 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4221 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4141 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4221 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4221 +num rollout transitions: 50000, reward mean: 4.4495 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 999 | +| eval/episode_length_std | 10.3 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.36 | +| loss/actor | -367 | +| loss/alpha | 0.00816 | +| loss/critic | 61.8 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.57 | +| rollout_info/dynamics_penalty_mean | 0.137 | +| rollout_info/dynamics_penalty_std | 0.312 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4221 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4213 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4346 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.234 | +| loss/actor | -366 | +| loss/alpha | -0.0128 | +| loss/critic | 62 | +| penalty/lcb_mean | 1.71 | +| penalty/lcb_std | 2.58 | +| rollout_info/dynamics_penalty_mean | 0.132 | +| rollout_info/dynamics_penalty_std | 0.308 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4126 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4327 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.488 | +| loss/actor | -368 | +| loss/alpha | -0.00832 | +| loss/critic | 60.7 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.57 | +| rollout_info/dynamics_penalty_mean | 0.136 | +| rollout_info/dynamics_penalty_std | 0.312 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4216 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4115 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4173 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4211 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 997 | +| eval/episode_length_std | 30.7 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 3.77 | +| loss/actor | -368 | +| loss/alpha | 0.0141 | +| loss/critic | 62.1 | +| penalty/lcb_mean | 1.72 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0.139 | +| rollout_info/dynamics_penalty_std | 0.315 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4169 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4169 +num rollout transitions: 50000, reward mean: 4.4183 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4289 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.287 | +| loss/actor | -368 | +| loss/alpha | -0.0034 | +| loss/critic | 62.5 | +| penalty/lcb_mean | 1.72 | +| penalty/lcb_std | 2.62 | +| rollout_info/dynamics_penalty_mean | 0.139 | +| rollout_info/dynamics_penalty_std | 0.315 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4197 +num rollout transitions: 50000, reward mean: 4.4136 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4155 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4199 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4255 +---------------------------------------------------------------------------------- +| alpha | 0.112 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.339 | +| loss/actor | -361 | +| loss/alpha | -0.00979 | +| loss/critic | 63.6 | +| penalty/lcb_mean | 1.75 | +| penalty/lcb_std | 2.62 | +| rollout_info/dynamics_penalty_mean | 0.14 | +| rollout_info/dynamics_penalty_std | 0.316 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 124551.41s diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_42×tamp_26-0317-003138/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_42×tamp_26-0317-003138/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_42×tamp_26-0317-003138/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_42×tamp_26-0317-003138/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..326c5d57d0f8937979ab37ca494c83338a55d03b --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_42×tamp_26-0317-003138/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_realnvp.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 42, "device": "cuda:7", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_42×tamp_26-0112-030600/model", "reward_penalty_coef": 0.5, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/realnvp", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "realnvp", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "realnvp"} diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_42×tamp_26-0317-003138/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_42×tamp_26-0317-003138/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..4803743e62a09cd8196160be2593facc4c8019e8 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_42×tamp_26-0317-003138/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.26605492127717373,156.93,58.36938495478602,4.542354513881584,4.989952380836074,-277.4259513259352,-1.7632172334739395,21.280560189414835,0.996729679890086,1.2584403467939798,0.3099010468484082,0.3681841104519618,50000.0,4.365718478583782,50000 +0.15303562667489004,40.02,0.39949968710876366,0.5321359219810375,-0.020377570896996704,-425.0460090209964,0.010122280052192598,38.14987878179572,1.3412561458683194,1.8892775609207146,0.15070981321627472,0.32157769161631755,50000.0,4.428560156119207,100000 +0.14895028140544614,174.43,107.48071966636618,9.352961431818237,11.244739394943432,-424.6890953094493,-0.01326998553074928,37.83468426448834,1.3067274721550837,1.8559291649734846,0.15671784844333972,0.32875723987865574,50000.0,4.432396055882204,150000 +0.14672402627766437,430.39,181.75251827691415,38.71738804682882,19.941516835605174,-417.88315031066367,0.013180364193469537,35.80030852079368,1.2727699662256053,1.7922928069448476,0.1504087683222769,0.322858448074317,50000.0,4.436137912740034,200000 +0.16568598434925164,570.71,298.7462232397257,52.890984199147105,32.601653329136006,-395.5486353692608,0.003729321618415261,37.17784193075203,1.3005598311591124,1.7768874664783545,0.14603271550707045,0.32079158392233353,50000.0,4.437464121134459,250000 +0.17015982652366027,1000.0,0.0,95.67623110867231,0.9806833991580893,-378.68231786865175,0.006977063624505046,42.663487229690126,1.370894021673213,1.9107770915508355,0.14017636914355286,0.3149459342691505,50000.0,4.435835975163976,300000 +0.17088226700425177,876.24,191.81814929771372,89.69297554751907,21.194786051945847,-372.7788122338886,-0.008653724323809227,44.88352459823685,1.3827804146110856,1.987985410546074,0.13589470721055028,0.3104662764117032,50000.0,4.440614971849644,350000 +0.16789076419413004,964.81,99.84625130669653,100.0039225486594,11.285289683850165,-373.23698696716366,0.0018129496482014746,47.591315552100944,1.3971602885878196,2.068703756060626,0.13104486293124998,0.3049988153263508,50000.0,4.436397532835842,400000 +0.16241071169942778,1000.0,0.0,99.32392166808891,1.8726277693842965,-372.08757835571134,-0.003275742669217294,49.177983232879754,1.439330410292151,2.168388188159481,0.12777753582095,0.30116482349735274,50000.0,4.4363140175419415,450000 +0.167761750864985,996.43,28.076771538052597,102.38300666484047,4.043769489015955,-369.2860809149211,0.006794252206124247,51.46543342470204,1.4679410846280978,2.184923214259195,0.12076989463083682,0.29320678103755143,50000.0,4.437497167520182,500000 +0.16576766952901997,996.15,30.495368500806794,107.4809008550605,4.037180138781926,-369.94916037841864,-0.0052676829705014475,51.05966512937562,1.4823498628163092,2.1742400248718288,0.12033903965911276,0.2929576457123752,50000.0,4.438990069899852,550000 +0.15931808430940048,994.6,43.76711093960852,106.77275227123208,5.322945437873499,-372.595869788211,-0.0017795369220898172,51.68511186717963,1.4705795631682725,2.2187837696588013,0.12197025641952071,0.29449035699490134,50000.0,4.435878131108535,600000 +0.1569682297992705,1000.0,0.0,102.46045152036093,1.6940542703538366,-370.59238375854426,-0.005835632585287073,52.00740016933378,1.4793166274237781,2.236455481247917,0.12158557165579317,0.2939346393723108,50000.0,4.436718558060494,650000 +0.14778593506157514,1000.0,0.0,107.71008834568275,0.5591590100782343,-369.5024272625776,-0.007325813252404354,49.76122183115016,1.4465571562039798,2.18923629264477,0.12267949040744267,0.29605935875861394,50000.0,4.436610637313095,700000 +0.15057687620609872,992.81,36.70441254127356,105.20136293436228,4.1402188538248925,-369.0157123559561,0.0058047698876634215,50.15949786415118,1.4848389097786157,2.237889230243003,0.12159256312541002,0.2941178494357437,50000.0,4.439247099957186,750000 +0.14886297268986504,1000.0,0.0,108.36013392853914,0.37402698086445396,-370.95666968262,0.00022633457072071566,50.97783748804027,1.4939298622274553,2.241965268853932,0.12213961037394677,0.2946660420870957,50000.0,4.439441065625841,800000 +0.1475423899310825,998.51,14.82531281288863,113.76832467038392,1.904831359971299,-370.18042718871675,0.00042097517464307303,52.0223352771183,1.4948207300841627,2.2839794143629377,0.11835153969593165,0.2909288077185086,50000.0,4.440221009677396,850000 +0.15156704630881493,1000.0,0.0,111.53797164895876,0.9018468993159852,-370.3712721136394,-0.004617570065297123,51.59204125684761,1.5216791997957264,2.249752172895682,0.1282983139813158,0.3018227594264904,50000.0,4.433182977320974,900000 +0.14466533627241984,991.93,28.82542454153971,113.81450200987832,3.5849217996838316,-372.87852213683715,0.014361861662473375,51.33653838174823,1.5144658625292953,2.291691114308815,0.13314324388214946,0.30656377596101114,50000.0,4.428356921688248,950000 +0.15138991081237624,960.51,105.32934016692595,108.93184458810117,12.958327212424049,-372.32194587402466,-0.010470384034179134,53.99269924631163,1.5426498185563056,2.3301750654148483,0.13146512097911228,0.3052509063308954,50000.0,4.429250563524819,1000000 +0.1439432393509134,989.76,36.75571248119127,111.78291056637273,4.618718377498402,-371.7854359637477,-1.3206404522008677e-05,51.38475594549157,1.5091204466462271,2.2694977232622753,0.12564573084856054,0.29892349801594775,50000.0,4.432597144751122,1050000 +0.1359560734668359,1000.0,0.0,112.0656609113549,0.6519042527377159,-375.8854442260722,-2.815443843598104e-05,50.980193651962004,1.47227173849702,2.286748609117298,0.132152890443722,0.3057067734533675,50000.0,4.430509210972963,1100000 +0.13845849128037635,1000.0,0.0,110.11951670792838,0.6353431296900318,-373.99526500610426,-0.0016904951678589294,53.838142691459566,1.5113368122816082,2.346407686887968,0.13084953614055614,0.30465225426406994,50000.0,4.431884410896555,1150000 +0.14041155046582296,964.46,105.26617880402041,105.40192643574169,12.600879666238763,-371.707917525632,0.004451100336574012,53.039552214584525,1.5119392042374797,2.3289889944958393,0.13046054650851038,0.30399703632789066,50000.0,4.432059467118656,1200000 +0.14034406536340788,977.67,78.42143265715055,110.85911287683834,9.84400344478045,-369.8239376165743,-0.0032217255821080253,54.073183166714294,1.5475611779212652,2.328327934142398,0.13138376019479234,0.3052746984916563,50000.0,4.432755966373038,1250000 +0.1385516057914486,999.41,5.870425878929058,105.8670700186902,1.816246017895028,-373.39689807067947,0.00024202785469591294,53.51182107511543,1.5417091791534396,2.3400558915972223,0.14022923834492604,0.3134071829949062,50000.0,4.42507634617497,1300000 +0.1384426650872854,910.13,151.3170614967129,101.81539862596252,18.360643936747024,-375.43750233031585,-0.0013661680245771178,56.04784325689285,1.5720472718691925,2.400231799651331,0.13419604374817362,0.3076836640816524,50000.0,4.431759816174829,1350000 +0.1379325716775638,846.71,179.52617051561032,95.4832506915193,22.410819537679643,-376.636457127074,0.000404081804267985,59.65887202234352,1.6036594350433282,2.4915573866343617,0.14336294469299157,0.3175162387679898,50000.0,4.423625811758659,1400000 +0.1411875358763345,1000.0,0.0,113.9776338969939,0.7542085396537173,-375.876662912598,-0.0040394599295780756,58.37804704605115,1.596591576986292,2.4563858895301633,0.1401951758358653,0.3141196898611297,50000.0,4.42686282712166,1450000 +0.13505774190753703,993.28,39.11651313703716,113.22331669885219,4.983987009544589,-378.5396825140393,0.001145180507227695,56.25966367675737,1.5463803083133631,2.420460904334755,0.132730424309923,0.30661582730167275,50000.0,4.433691048624649,1500000 +0.13740298378765498,995.71,29.12603474556741,109.88484192750813,3.5032220142095785,-376.20029509704716,-0.00501638571497061,56.5110379467771,1.5772885686946267,2.4069844146514128,0.13562423603803928,0.30885081298913075,50000.0,4.434641381542706,1550000 +0.1339142448373116,536.48,220.9926912818612,53.64194313638456,25.981759240853574,-372.6245643127482,-0.0004475465397537001,54.02424536335044,1.5534977885985222,2.3632316537690037,0.14412709411137797,0.3178179171314126,50000.0,4.4266117545371335,1600000 +0.1333900619927054,732.34,171.5635287582999,79.48519109731457,19.983757400678208,-373.5680649548322,0.006142325503453617,55.90962682087044,1.5749215002679882,2.39881420061111,0.1465042113068678,0.319736214796607,50000.0,4.425924758360346,1650000 +0.14097744297832604,940.78,133.17594227186828,103.54680125624003,15.721549434582736,-373.31484402649284,0.003117588142640877,57.173387441940555,1.5956941570234169,2.4192465486693293,0.14018665924860257,0.31451462746655806,50000.0,4.428123765809087,1700000 +0.13233955430373923,1000.0,0.0,110.79707563669076,0.970167262156275,-376.15634131835475,-0.008861106479055992,56.72956522256891,1.5799369965553325,2.4469303742337587,0.1470236808916842,0.32111764083718186,50000.0,4.427550367087373,1750000 +0.13011303166717184,989.61,47.3188958028397,109.21292825095752,6.233695559509781,-376.00492933043716,-0.007144857867471864,57.69541893304792,1.5993177226018844,2.4776337209153465,0.14979802747520912,0.3243115883802093,50000.0,4.42425691419219,1800000 +0.129908873200567,737.72,220.52678204698856,81.48528615026034,26.839051857601266,-374.0715279180887,0.0062470700731873646,59.663970365124214,1.6312522488284058,2.504405088255411,0.14847732570662142,0.32312452608653136,50000.0,4.425848203403254,1850000 +0.13755815329551677,865.11,171.1016011029704,98.42881375672957,21.188222205488614,-373.88336623962795,0.007412978801541155,60.36784553974147,1.645281717412473,2.4774779118895514,0.14624619167364158,0.3207668494099496,50000.0,4.424688633064564,1900000 +0.13532326414644605,997.47,18.613143205810243,114.95320247560443,2.4258124413495628,-378.00603596680367,-0.006202763083837929,59.816669824466274,1.6268651595425534,2.5276599765634518,0.1412994749806552,0.3146974787892039,50000.0,4.428276219482791,1950000 +0.1338521673171211,996.83,31.541101756279847,114.13981551476039,3.961587624794103,-373.9923211981211,0.0008196563590689964,62.629835720806234,1.6407705270242794,2.586804074711783,0.13905694250159611,0.313312235893812,50000.0,4.427444010630698,2000000 +0.1321366691608714,994.63,53.430825372625485,114.06704247099117,6.608778154398232,-373.2077558905081,-0.005233704830594324,60.79836281291982,1.6310707297659293,2.546799522140038,0.13845867169333456,0.31240935135229614,50000.0,4.430345936623055,2050000 +0.12772573870122558,884.54,163.5521580414028,99.30791962581532,19.794781537978086,-372.8101725701855,-0.008277933859750659,59.49771172079086,1.615939132034794,2.513000608887675,0.1460159002534534,0.3192706448802604,50000.0,4.426273326629537,2100000 +0.13151272392004681,984.5,72.1379927638689,113.49169444992422,9.44034134880221,-372.33095251953046,0.014449783519953855,60.843453228588665,1.6331112293267134,2.5353328451728583,0.1395201820964239,0.31336031881602144,50000.0,4.4284058863667415,2150000 +0.1360692681343866,954.67,108.50124930156335,108.13432717540744,13.360514825577386,-367.5481362548797,-0.009276209551468448,58.72297087385191,1.632869548437605,2.493110756900345,0.1335222948190094,0.3073677302404752,50000.0,4.430078081151067,2200000 +0.13009164486810745,1000.0,0.0,115.3847150488969,0.5995900704786546,-370.4082616058368,0.0025330347030238943,58.6617711626249,1.612181883287435,2.524464865028812,0.13864035352022538,0.3129371772741651,50000.0,4.429131986217777,2250000 +0.13325294409111207,621.84,278.8541095268277,67.36540911939916,35.31694266167032,-368.1287585565116,0.003394766258746503,62.89433930717474,1.6808736839222993,2.617610327613374,0.13335009801839326,0.3073743541216873,50000.0,4.4283207239078015,2300000 +0.1380461654059591,955.46,104.23822907167981,109.12057471389409,12.880267848351037,-361.7807469989034,0.00650853668261323,62.93156051555712,1.6974396363425348,2.5927456236553788,0.13206968683220444,0.30626395735006273,50000.0,4.432708591990715,2350000 +0.13203544403970258,979.23,93.05674129261135,111.61033841851258,11.481681783526366,-363.2417703826922,-0.0004426832209154884,62.82493973726274,1.6835156494832242,2.5947443230485776,0.13171035711607118,0.30531786630535324,50000.0,4.430110854217702,2400000 +0.12787337662145581,1000.0,0.0,115.93198576407524,0.35274020962813313,-366.91526944152724,-0.011086044285874792,61.857768573036566,1.6513063053035806,2.5988044619226573,0.13005868428745337,0.30371267104391114,50000.0,4.433970154092462,2450000 +0.1308986552287618,977.62,86.81748441414321,112.21255033677657,10.89427700735945,-366.4342581201191,0.008435182552412266,61.69160472095503,1.657808216872221,2.58350433776377,0.13113107907133437,0.30539593892890077,50000.0,4.434647635411689,2500000 +0.1350045913484713,1000.0,0.0,116.86360317256299,0.5076540586633635,-371.4752222515831,-0.0076792234714328925,62.96147479118334,1.6910090309786687,2.6027571787500676,0.13449093780073437,0.3085329071340136,50000.0,4.429984118363627,2550000 +0.1365883636142308,1000.0,0.0,115.0209508322337,0.7416257720418871,-366.6764120867875,0.001839718405790471,60.87110544982904,1.6668709448146866,2.535173261330158,0.13329470978493893,0.30826560323784236,50000.0,4.434482250735831,2600000 +0.13307126619219503,1000.0,0.0,118.27705951903249,0.9185009164348534,-365.1354779345722,0.0028185198314488397,58.674253111820384,1.6351998085927963,2.494546874530282,0.13066869045231266,0.30562709148962836,50000.0,4.436101763871306,2650000 +0.12749350624129258,1000.0,0.0,113.70910621968821,0.3480569970714656,-369.4975663940419,-0.004158218044880748,59.692224111271344,1.6661049912690888,2.5323609310746042,0.13476461646679572,0.309482049149227,50000.0,4.4317873838522335,2700000 +0.12723375185474997,998.96,10.347869345908844,113.31730684631871,1.357733476716994,-366.7230713159198,0.008162241811379704,61.83090334062646,1.7007074109720992,2.5662988950276517,0.13670113931705707,0.31185837642998465,50000.0,4.428689547295626,2750000 +0.13237810011416637,1000.0,0.0,113.7800215986051,0.2341662145960299,-366.3905244390809,-0.012806721388883764,62.03151214096089,1.7122347105240778,2.5812358007383005,0.13217984876848995,0.30824114168551464,50000.0,4.433363477052729,2800000 +0.11690181254491272,1000.0,0.0,113.46832131551585,0.4884391349939462,-367.91748516845564,-0.008322285961062105,60.70984064065959,1.6755400584316158,2.570632928688556,0.1363235730221595,0.3119310742455971,50000.0,4.43382271749113,2850000 +0.12456861351534679,996.91,30.745111806594554,112.98446588028636,3.770882995278862,-368.37434360961134,0.014110064838342979,62.0830224247551,1.7193839479899367,2.6079880535172895,0.13905233951828833,0.31524652559952715,50000.0,4.4277798035523634,2900000 +0.11718622937291968,1000.0,0.0,114.34221410245999,0.28669963334967835,-367.7101440936233,-0.0034018324407935245,62.52645680499985,1.715606765274974,2.618981514627883,0.13928208034090883,0.31527640178548083,50000.0,4.428960654261393,2950000 +0.11161521209150593,1000.0,0.0,114.5223443801261,0.3392172646751272,-361.09071501648157,-0.009794735929555815,63.55887331085283,1.7458939349079088,2.622994779865748,0.13970478958665564,0.31553286212681525,50000.0,4.428624196176671,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_456×tamp_26-0319-225151/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_456×tamp_26-0319-225151/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..5a2ce34841e0d725a7aaf5198bfcd13cfb49d0c1 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_456×tamp_26-0319-225151/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 3.9553 +num rollout transitions: 50000, reward mean: 4.1184 +num rollout transitions: 50000, reward mean: 4.1695 +num rollout transitions: 50000, reward mean: 4.1490 +num rollout transitions: 50000, reward mean: 4.2006 +num rollout transitions: 50000, reward mean: 4.1773 +num rollout transitions: 50000, reward mean: 4.1688 +num rollout transitions: 50000, reward mean: 4.2145 +num rollout transitions: 50000, reward mean: 4.2638 +num rollout transitions: 50000, reward mean: 4.3018 +num rollout transitions: 50000, reward mean: 4.3603 +num rollout transitions: 50000, reward mean: 4.3703 +num rollout transitions: 50000, reward mean: 4.3589 +num rollout transitions: 50000, reward mean: 4.3685 +num rollout transitions: 50000, reward mean: 4.4002 +num rollout transitions: 50000, reward mean: 4.4045 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.3121 +num rollout transitions: 50000, reward mean: 4.3489 +num rollout transitions: 50000, reward mean: 4.3985 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4626 +num rollout transitions: 50000, reward mean: 4.4513 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4553 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4576 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4519 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4668 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4517 +num rollout transitions: 50000, reward mean: 4.4724 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4330 +---------------------------------------------------------------------------------- +| alpha | 0.263 | +| eval/episode_length | 71.5 | +| eval/episode_length_std | 7.86 | +| eval/normalized_episode_reward | 3.32 | +| eval/normalized_episode_reward_std | 0.373 | +| loss/actor | -282 | +| loss/alpha | -1.6 | +| loss/critic | 19.4 | +| penalty/lcb_mean | 0.937 | +| penalty/lcb_std | 1.19 | +| rollout_info/dynamics_penalty_mean | 0.299 | +| rollout_info/dynamics_penalty_std | 0.361 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.38 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4128 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4154 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4547 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4541 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.3937 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4158 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.3777 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4162 +num rollout transitions: 50000, reward mean: 4.4063 +num rollout transitions: 50000, reward mean: 4.4148 +num rollout transitions: 50000, reward mean: 4.4117 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.3934 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4149 +num rollout transitions: 50000, reward mean: 4.4158 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4109 +num rollout transitions: 50000, reward mean: 4.4024 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4182 +num rollout transitions: 50000, reward mean: 4.4101 +num rollout transitions: 50000, reward mean: 4.4052 +num rollout transitions: 50000, reward mean: 4.4040 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4386 +---------------------------------------------------------------------------------- +| alpha | 0.16 | +| eval/episode_length | 178 | +| eval/episode_length_std | 77.8 | +| eval/normalized_episode_reward | 1.77 | +| eval/normalized_episode_reward_std | 5.96 | +| loss/actor | -433 | +| loss/alpha | 0.00117 | +| loss/critic | 34.4 | +| penalty/lcb_mean | 1.21 | +| penalty/lcb_std | 1.78 | +| rollout_info/dynamics_penalty_mean | 0.153 | +| rollout_info/dynamics_penalty_std | 0.324 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.42 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4195 +num rollout transitions: 50000, reward mean: 4.4066 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4660 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4590 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4534 +num rollout transitions: 50000, reward mean: 4.4525 +num rollout transitions: 50000, reward mean: 4.4619 +num rollout transitions: 50000, reward mean: 4.4532 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4494 +num rollout transitions: 50000, reward mean: 4.4672 +num rollout transitions: 50000, reward mean: 4.4637 +num rollout transitions: 50000, reward mean: 4.4672 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4441 +num rollout transitions: 50000, reward mean: 4.4543 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4578 +num rollout transitions: 50000, reward mean: 4.4666 +num rollout transitions: 50000, reward mean: 4.4527 +num rollout transitions: 50000, reward mean: 4.4685 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4544 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4585 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4658 +---------------------------------------------------------------------------------- +| alpha | 0.15 | +| eval/episode_length | 683 | +| eval/episode_length_std | 244 | +| eval/normalized_episode_reward | 62.9 | +| eval/normalized_episode_reward_std | 24.5 | +| loss/actor | -436 | +| loss/alpha | -0.00843 | +| loss/critic | 31.3 | +| penalty/lcb_mean | 1.13 | +| penalty/lcb_std | 1.74 | +| rollout_info/dynamics_penalty_mean | 0.138 | +| rollout_info/dynamics_penalty_std | 0.311 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.45 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4503 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4566 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4616 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4546 +num rollout transitions: 50000, reward mean: 4.4638 +num rollout transitions: 50000, reward mean: 4.4618 +num rollout transitions: 50000, reward mean: 4.4679 +num rollout transitions: 50000, reward mean: 4.4668 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4095 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4555 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4649 +---------------------------------------------------------------------------------- +| alpha | 0.148 | +| eval/episode_length | 92.6 | +| eval/episode_length_std | 4.83 | +| eval/normalized_episode_reward | 4.02 | +| eval/normalized_episode_reward_std | 0.159 | +| loss/actor | -429 | +| loss/alpha | 0.0115 | +| loss/critic | 28.2 | +| penalty/lcb_mean | 1.11 | +| penalty/lcb_std | 1.58 | +| rollout_info/dynamics_penalty_mean | 0.144 | +| rollout_info/dynamics_penalty_std | 0.317 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4547 +num rollout transitions: 50000, reward mean: 4.4647 +num rollout transitions: 50000, reward mean: 4.4665 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4524 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4486 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4493 +num rollout transitions: 50000, reward mean: 4.4545 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4500 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4415 +----------------------------------------------------------------------------------- +| alpha | 0.155 | +| eval/episode_length | 142 | +| eval/episode_length_std | 15.2 | +| eval/normalized_episode_reward | 7.13 | +| eval/normalized_episode_reward_std | 1.73 | +| loss/actor | -424 | +| loss/alpha | -0.000244 | +| loss/critic | 31.3 | +| penalty/lcb_mean | 1.17 | +| penalty/lcb_std | 1.68 | +| rollout_info/dynamics_penalty_mean | 0.137 | +| rollout_info/dynamics_penalty_std | 0.309 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 250000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4119 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4545 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4490 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4581 +num rollout transitions: 50000, reward mean: 4.4701 +num rollout transitions: 50000, reward mean: 4.4605 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4528 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4468 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4348 +----------------------------------------------------------------------------------- +| alpha | 0.16 | +| eval/episode_length | 956 | +| eval/episode_length_std | 134 | +| eval/normalized_episode_reward | 95 | +| eval/normalized_episode_reward_std | 14.1 | +| loss/actor | -412 | +| loss/alpha | -0.000712 | +| loss/critic | 36.5 | +| penalty/lcb_mean | 1.25 | +| penalty/lcb_std | 1.84 | +| rollout_info/dynamics_penalty_mean | 0.133 | +| rollout_info/dynamics_penalty_std | 0.306 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 300000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4637 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4167 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.3854 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4550 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4529 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4521 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4171 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4199 +num rollout transitions: 50000, reward mean: 4.4228 +---------------------------------------------------------------------------------- +| alpha | 0.156 | +| eval/episode_length | 977 | +| eval/episode_length_std | 102 | +| eval/normalized_episode_reward | 96.9 | +| eval/normalized_episode_reward_std | 10.9 | +| loss/actor | -403 | +| loss/alpha | -0.00226 | +| loss/critic | 37.7 | +| penalty/lcb_mean | 1.28 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 0.13 | +| rollout_info/dynamics_penalty_std | 0.303 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4622 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4509 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4529 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4449 +---------------------------------------------------------------------------------- +| alpha | 0.16 | +| eval/episode_length | 780 | +| eval/episode_length_std | 281 | +| eval/normalized_episode_reward | 77.7 | +| eval/normalized_episode_reward_std | 29.7 | +| loss/actor | -401 | +| loss/alpha | 0.00336 | +| loss/critic | 43.3 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.03 | +| rollout_info/dynamics_penalty_mean | 0.128 | +| rollout_info/dynamics_penalty_std | 0.301 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4526 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4552 +num rollout transitions: 50000, reward mean: 4.4493 +num rollout transitions: 50000, reward mean: 4.4569 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4164 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4625 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4526 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4585 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4515 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4541 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4184 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4257 +---------------------------------------------------------------------------------- +| alpha | 0.153 | +| eval/episode_length | 142 | +| eval/episode_length_std | 4.67 | +| eval/normalized_episode_reward | 7.95 | +| eval/normalized_episode_reward_std | 0.583 | +| loss/actor | -396 | +| loss/alpha | -0.00707 | +| loss/critic | 43 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 2.09 | +| rollout_info/dynamics_penalty_mean | 0.126 | +| rollout_info/dynamics_penalty_std | 0.298 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4153 +num rollout transitions: 50000, reward mean: 4.4602 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4517 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4164 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4549 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4552 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4384 +---------------------------------------------------------------------------------- +| alpha | 0.146 | +| eval/episode_length | 132 | +| eval/episode_length_std | 13.9 | +| eval/normalized_episode_reward | 6.67 | +| eval/normalized_episode_reward_std | 0.982 | +| loss/actor | -396 | +| loss/alpha | -0.00564 | +| loss/critic | 43.2 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 2.1 | +| rollout_info/dynamics_penalty_mean | 0.13 | +| rollout_info/dynamics_penalty_std | 0.302 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4111 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4594 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4585 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4568 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4647 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4551 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4526 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4484 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4572 +num rollout transitions: 50000, reward mean: 4.4662 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4373 +---------------------------------------------------------------------------------- +| alpha | 0.151 | +| eval/episode_length | 995 | +| eval/episode_length_std | 35.8 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 4.55 | +| loss/actor | -395 | +| loss/alpha | 0.0072 | +| loss/critic | 44.5 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.13 | +| rollout_info/dynamics_penalty_mean | 0.126 | +| rollout_info/dynamics_penalty_std | 0.298 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4176 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4566 +num rollout transitions: 50000, reward mean: 4.4069 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4176 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4002 +num rollout transitions: 50000, reward mean: 4.3807 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4146 +num rollout transitions: 50000, reward mean: 4.4221 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4494 +num rollout transitions: 50000, reward mean: 4.4168 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4388 +---------------------------------------------------------------------------------- +| alpha | 0.151 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.886 | +| loss/actor | -393 | +| loss/alpha | -0.00172 | +| loss/critic | 47.2 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 0.129 | +| rollout_info/dynamics_penalty_std | 0.301 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4199 +num rollout transitions: 50000, reward mean: 4.4196 +num rollout transitions: 50000, reward mean: 4.4128 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4079 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4140 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4175 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4073 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4523 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4153 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4158 +---------------------------------------------------------------------------------- +| alpha | 0.147 | +| eval/episode_length | 928 | +| eval/episode_length_std | 197 | +| eval/normalized_episode_reward | 90 | +| eval/normalized_episode_reward_std | 20.6 | +| loss/actor | -393 | +| loss/alpha | 0.00607 | +| loss/critic | 46.3 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.17 | +| rollout_info/dynamics_penalty_mean | 0.126 | +| rollout_info/dynamics_penalty_std | 0.299 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4075 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4585 +num rollout transitions: 50000, reward mean: 4.4230 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4192 +num rollout transitions: 50000, reward mean: 4.4143 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4079 +num rollout transitions: 50000, reward mean: 4.4180 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4105 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4173 +num rollout transitions: 50000, reward mean: 4.4120 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4132 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4132 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4341 +---------------------------------------------------------------------------------- +| alpha | 0.15 | +| eval/episode_length | 991 | +| eval/episode_length_std | 47.1 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 5.85 | +| loss/actor | -392 | +| loss/alpha | -0.00853 | +| loss/critic | 49 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.25 | +| rollout_info/dynamics_penalty_mean | 0.13 | +| rollout_info/dynamics_penalty_std | 0.302 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4051 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4538 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4312 +---------------------------------------------------------------------------------- +| alpha | 0.146 | +| eval/episode_length | 983 | +| eval/episode_length_std | 75.7 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 9.11 | +| loss/actor | -388 | +| loss/alpha | 0.00212 | +| loss/critic | 47.3 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.2 | +| rollout_info/dynamics_penalty_mean | 0.122 | +| rollout_info/dynamics_penalty_std | 0.295 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4181 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4174 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4098 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4574 +---------------------------------------------------------------------------------- +| alpha | 0.148 | +| eval/episode_length | 995 | +| eval/episode_length_std | 37.7 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 4.86 | +| loss/actor | -391 | +| loss/alpha | 0.00228 | +| loss/critic | 47.6 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 0.126 | +| rollout_info/dynamics_penalty_std | 0.298 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4504 +num rollout transitions: 50000, reward mean: 4.4698 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4539 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4517 +num rollout transitions: 50000, reward mean: 4.4614 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4640 +num rollout transitions: 50000, reward mean: 4.4578 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4360 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 1.34 | +| loss/actor | -393 | +| loss/alpha | -0.00714 | +| loss/critic | 46.4 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.18 | +| rollout_info/dynamics_penalty_mean | 0.122 | +| rollout_info/dynamics_penalty_std | 0.294 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4645 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4296 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4155 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4160 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4554 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4531 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4255 +num rollout transitions: 50000, reward mean: 4.4295 +---------------------------------------------------------------------------------- +| alpha | 0.147 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 1.12 | +| loss/actor | -394 | +| loss/alpha | 0.00177 | +| loss/critic | 48 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.23 | +| rollout_info/dynamics_penalty_mean | 0.124 | +| rollout_info/dynamics_penalty_std | 0.297 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4517 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4441 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4546 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4525 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4524 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4569 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4441 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4418 +---------------------------------------------------------------------------------- +| alpha | 0.146 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 1.48 | +| loss/actor | -392 | +| loss/alpha | 0.00415 | +| loss/critic | 47.4 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 0.12 | +| rollout_info/dynamics_penalty_std | 0.292 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4568 +num rollout transitions: 50000, reward mean: 4.4441 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4579 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4130 +num rollout transitions: 50000, reward mean: 4.4516 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4547 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4527 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4639 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4178 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4504 +num rollout transitions: 50000, reward mean: 4.4519 +num rollout transitions: 50000, reward mean: 4.4529 +num rollout transitions: 50000, reward mean: 4.4517 +num rollout transitions: 50000, reward mean: 4.4587 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4467 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4528 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4523 +---------------------------------------------------------------------------------- +| alpha | 0.146 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 0.361 | +| loss/actor | -388 | +| loss/alpha | -0.00126 | +| loss/critic | 48.3 | +| penalty/lcb_mean | 1.43 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 0.118 | +| rollout_info/dynamics_penalty_std | 0.29 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4113 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4481 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4499 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4532 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4173 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4486 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4605 +num rollout transitions: 50000, reward mean: 4.4504 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4454 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 1.83 | +| loss/actor | -389 | +| loss/alpha | 0.00588 | +| loss/critic | 50.3 | +| penalty/lcb_mean | 1.45 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 0.118 | +| rollout_info/dynamics_penalty_std | 0.29 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4489 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4493 +num rollout transitions: 50000, reward mean: 4.4569 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4597 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4613 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4481 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4577 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4191 +---------------------------------------------------------------------------------- +| alpha | 0.146 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 0.327 | +| loss/actor | -389 | +| loss/alpha | -0.0128 | +| loss/critic | 51.2 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.29 | +| rollout_info/dynamics_penalty_mean | 0.122 | +| rollout_info/dynamics_penalty_std | 0.294 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4639 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4130 +num rollout transitions: 50000, reward mean: 4.4176 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4599 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4513 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4441 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4507 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4455 +---------------------------------------------------------------------------------- +| alpha | 0.143 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.834 | +| loss/actor | -391 | +| loss/alpha | 0.00344 | +| loss/critic | 51.2 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 0.125 | +| rollout_info/dynamics_penalty_std | 0.296 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4542 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4534 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4535 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4553 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4599 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4570 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4171 +num rollout transitions: 50000, reward mean: 4.4071 +num rollout transitions: 50000, reward mean: 4.4167 +num rollout transitions: 50000, reward mean: 4.4061 +num rollout transitions: 50000, reward mean: 4.4130 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4453 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 0.675 | +| loss/actor | -393 | +| loss/alpha | 0.00544 | +| loss/critic | 50.5 | +| penalty/lcb_mean | 1.43 | +| penalty/lcb_std | 2.32 | +| rollout_info/dynamics_penalty_mean | 0.125 | +| rollout_info/dynamics_penalty_std | 0.296 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4298 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4166 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4539 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4558 +num rollout transitions: 50000, reward mean: 4.4523 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4535 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4204 +num rollout transitions: 50000, reward mean: 4.4490 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4596 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4184 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4209 +num rollout transitions: 50000, reward mean: 4.4328 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.383 | +| loss/actor | -392 | +| loss/alpha | -0.0041 | +| loss/critic | 52.1 | +| penalty/lcb_mean | 1.45 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 0.124 | +| rollout_info/dynamics_penalty_std | 0.296 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4585 +num rollout transitions: 50000, reward mean: 4.4565 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4686 +num rollout transitions: 50000, reward mean: 4.4524 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4653 +num rollout transitions: 50000, reward mean: 4.4511 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4457 +num rollout transitions: 50000, reward mean: 4.4523 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4091 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4440 +---------------------------------------------------------------------------------- +| alpha | 0.142 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.35 | +| loss/actor | -393 | +| loss/alpha | 0.00436 | +| loss/critic | 50.1 | +| penalty/lcb_mean | 1.45 | +| penalty/lcb_std | 2.29 | +| rollout_info/dynamics_penalty_mean | 0.125 | +| rollout_info/dynamics_penalty_std | 0.297 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4510 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4539 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4605 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4532 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4374 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4136 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4493 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4580 +---------------------------------------------------------------------------------- +| alpha | 0.142 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 1.91 | +| loss/actor | -391 | +| loss/alpha | -0.00927 | +| loss/critic | 52.2 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.127 | +| rollout_info/dynamics_penalty_std | 0.299 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4558 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4627 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4510 +num rollout transitions: 50000, reward mean: 4.4489 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4529 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4579 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4540 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4081 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4494 +num rollout transitions: 50000, reward mean: 4.4179 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 885 | +| eval/episode_length_std | 199 | +| eval/normalized_episode_reward | 93.5 | +| eval/normalized_episode_reward_std | 23.1 | +| loss/actor | -393 | +| loss/alpha | -0.00114 | +| loss/critic | 49.6 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.3 | +| rollout_info/dynamics_penalty_mean | 0.125 | +| rollout_info/dynamics_penalty_std | 0.296 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4547 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4064 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4121 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4089 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4166 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4254 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4369 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.664 | +| loss/actor | -394 | +| loss/alpha | 0.00236 | +| loss/critic | 53.6 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0.131 | +| rollout_info/dynamics_penalty_std | 0.303 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4567 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4215 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4140 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4543 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4565 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4575 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4104 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4510 +num rollout transitions: 50000, reward mean: 4.4526 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 991 | +| eval/episode_length_std | 60.3 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 7.21 | +| loss/actor | -391 | +| loss/alpha | 0.0037 | +| loss/critic | 53.7 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 0.125 | +| rollout_info/dynamics_penalty_std | 0.297 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4222 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4097 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4176 +num rollout transitions: 50000, reward mean: 4.4099 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4213 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4170 +num rollout transitions: 50000, reward mean: 4.4137 +num rollout transitions: 50000, reward mean: 4.4167 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4647 +num rollout transitions: 50000, reward mean: 4.4554 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4561 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 991 | +| eval/episode_length_std | 60.9 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 7.38 | +| loss/actor | -397 | +| loss/alpha | 0.00812 | +| loss/critic | 55.3 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.47 | +| rollout_info/dynamics_penalty_mean | 0.133 | +| rollout_info/dynamics_penalty_std | 0.304 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4464 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4202 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4123 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4548 +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4074 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4128 +---------------------------------------------------------------------------------- +| alpha | 0.137 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.188 | +| loss/actor | -396 | +| loss/alpha | -0.0193 | +| loss/critic | 57.9 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.52 | +| rollout_info/dynamics_penalty_mean | 0.133 | +| rollout_info/dynamics_penalty_std | 0.305 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4568 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4228 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4186 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4163 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4490 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4549 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4459 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 995 | +| eval/episode_length_std | 42.1 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 4.95 | +| loss/actor | -400 | +| loss/alpha | 0.00751 | +| loss/critic | 59.3 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0.137 | +| rollout_info/dynamics_penalty_std | 0.309 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4494 +num rollout transitions: 50000, reward mean: 4.4580 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4647 +num rollout transitions: 50000, reward mean: 4.4541 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4476 +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4603 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4664 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4571 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4606 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4511 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4592 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.524 | +| loss/actor | -397 | +| loss/alpha | 0.00145 | +| loss/critic | 56 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.51 | +| rollout_info/dynamics_penalty_mean | 0.129 | +| rollout_info/dynamics_penalty_std | 0.301 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4543 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4618 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4170 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4513 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4176 +num rollout transitions: 50000, reward mean: 4.4105 +num rollout transitions: 50000, reward mean: 4.4181 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4138 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4095 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4321 +num rollout transitions: 50000, reward mean: 4.4573 +num rollout transitions: 50000, reward mean: 4.4642 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4603 +num rollout transitions: 50000, reward mean: 4.4512 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4182 +num rollout transitions: 50000, reward mean: 4.4205 +num rollout transitions: 50000, reward mean: 4.4539 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4349 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4541 +num rollout transitions: 50000, reward mean: 4.4509 +num rollout transitions: 50000, reward mean: 4.4523 +num rollout transitions: 50000, reward mean: 4.4617 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4469 +----------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 1.32 | +| loss/actor | -396 | +| loss/alpha | -8.62e-06 | +| loss/critic | 56 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0.133 | +| rollout_info/dynamics_penalty_std | 0.305 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1750000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4521 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4545 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4513 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4069 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4533 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4539 +num rollout transitions: 50000, reward mean: 4.4519 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4416 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4619 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4388 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4425 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.592 | +| loss/actor | -394 | +| loss/alpha | 0.00432 | +| loss/critic | 56.5 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.5 | +| rollout_info/dynamics_penalty_mean | 0.13 | +| rollout_info/dynamics_penalty_std | 0.302 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4553 +num rollout transitions: 50000, reward mean: 4.4579 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4602 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4518 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4613 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4614 +num rollout transitions: 50000, reward mean: 4.4544 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4511 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4517 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4517 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4392 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4180 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4144 +num rollout transitions: 50000, reward mean: 4.4390 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.446 | +| loss/actor | -397 | +| loss/alpha | -0.00387 | +| loss/critic | 59 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.57 | +| rollout_info/dynamics_penalty_mean | 0.134 | +| rollout_info/dynamics_penalty_std | 0.306 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4526 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4148 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4122 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4154 +num rollout transitions: 50000, reward mean: 4.4135 +num rollout transitions: 50000, reward mean: 4.4097 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.3959 +num rollout transitions: 50000, reward mean: 4.4201 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4562 +num rollout transitions: 50000, reward mean: 4.4508 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4355 +---------------------------------------------------------------------------------- +| alpha | 0.144 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.207 | +| loss/actor | -396 | +| loss/alpha | 0.00779 | +| loss/critic | 61.1 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.6 | +| rollout_info/dynamics_penalty_mean | 0.136 | +| rollout_info/dynamics_penalty_std | 0.307 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4521 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.3986 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4576 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4234 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4035 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4374 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.335 | +| loss/actor | -399 | +| loss/alpha | 0.00331 | +| loss/critic | 61.9 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.142 | +| rollout_info/dynamics_penalty_std | 0.314 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4587 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4581 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4534 +num rollout transitions: 50000, reward mean: 4.4383 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4098 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4498 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4230 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4555 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4584 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4563 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4148 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4306 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.02 | +| loss/actor | -399 | +| loss/alpha | -0.0115 | +| loss/critic | 62.2 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.141 | +| rollout_info/dynamics_penalty_std | 0.313 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4652 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4176 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4139 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4333 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4545 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4147 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4293 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.351 | +| loss/actor | -398 | +| loss/alpha | -0.00203 | +| loss/critic | 58.9 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0.139 | +| rollout_info/dynamics_penalty_std | 0.312 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4491 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4538 +num rollout transitions: 50000, reward mean: 4.4503 +num rollout transitions: 50000, reward mean: 4.4557 +num rollout transitions: 50000, reward mean: 4.4492 +num rollout transitions: 50000, reward mean: 4.4040 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4484 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4565 +num rollout transitions: 50000, reward mean: 4.4477 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4536 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4529 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4515 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4525 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4481 +num rollout transitions: 50000, reward mean: 4.4470 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4309 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 925 | +| eval/episode_length_std | 130 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 16.1 | +| loss/actor | -399 | +| loss/alpha | -0.00302 | +| loss/critic | 59.2 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0.134 | +| rollout_info/dynamics_penalty_std | 0.306 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4515 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4520 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4444 +num rollout transitions: 50000, reward mean: 4.4521 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4506 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4564 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4191 +num rollout transitions: 50000, reward mean: 4.4123 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4129 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4434 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4172 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4171 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.216 | +| loss/actor | -398 | +| loss/alpha | 0.0127 | +| loss/critic | 58.1 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0.14 | +| rollout_info/dynamics_penalty_std | 0.312 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4587 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4211 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4065 +num rollout transitions: 50000, reward mean: 4.4083 +num rollout transitions: 50000, reward mean: 4.4181 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4161 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4370 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4132 +num rollout transitions: 50000, reward mean: 4.4128 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.3995 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4221 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 998 | +| eval/episode_length_std | 21.9 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 3.15 | +| loss/actor | -400 | +| loss/alpha | -0.0101 | +| loss/critic | 56.9 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.56 | +| rollout_info/dynamics_penalty_mean | 0.149 | +| rollout_info/dynamics_penalty_std | 0.321 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4263 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4143 +num rollout transitions: 50000, reward mean: 4.4124 +num rollout transitions: 50000, reward mean: 4.4092 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4244 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4497 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4404 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4102 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4240 +num rollout transitions: 50000, reward mean: 4.4122 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4294 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4061 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4161 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4450 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.623 | +| loss/actor | -399 | +| loss/alpha | 0.00693 | +| loss/critic | 59.6 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.143 | +| rollout_info/dynamics_penalty_std | 0.316 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4449 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4212 +num rollout transitions: 50000, reward mean: 4.4451 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4447 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4450 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4026 +num rollout transitions: 50000, reward mean: 4.4194 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4192 +num rollout transitions: 50000, reward mean: 4.3966 +num rollout transitions: 50000, reward mean: 4.4114 +num rollout transitions: 50000, reward mean: 4.4233 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4230 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4247 +num rollout transitions: 50000, reward mean: 4.4047 +num rollout transitions: 50000, reward mean: 4.4185 +num rollout transitions: 50000, reward mean: 4.4180 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4176 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.252 | +| loss/actor | -397 | +| loss/alpha | 0.00309 | +| loss/critic | 59.2 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.14 | +| rollout_info/dynamics_penalty_std | 0.313 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4164 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4167 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4238 +num rollout transitions: 50000, reward mean: 4.4117 +num rollout transitions: 50000, reward mean: 4.4119 +num rollout transitions: 50000, reward mean: 4.4366 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4485 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4487 +num rollout transitions: 50000, reward mean: 4.4430 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4146 +num rollout transitions: 50000, reward mean: 4.4137 +num rollout transitions: 50000, reward mean: 4.4050 +num rollout transitions: 50000, reward mean: 4.4184 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.3897 +num rollout transitions: 50000, reward mean: 4.4164 +num rollout transitions: 50000, reward mean: 4.4130 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4173 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4168 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4305 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4422 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4220 +num rollout transitions: 50000, reward mean: 4.4360 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4417 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4495 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.226 | +| loss/actor | -399 | +| loss/alpha | -0.0122 | +| loss/critic | 56.7 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.57 | +| rollout_info/dynamics_penalty_mean | 0.142 | +| rollout_info/dynamics_penalty_std | 0.314 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4154 +num rollout transitions: 50000, reward mean: 4.4217 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4175 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4484 +num rollout transitions: 50000, reward mean: 4.4517 +num rollout transitions: 50000, reward mean: 4.4465 +num rollout transitions: 50000, reward mean: 4.4469 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4458 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4248 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4226 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4123 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4448 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4270 +num rollout transitions: 50000, reward mean: 4.4188 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4463 +num rollout transitions: 50000, reward mean: 4.4483 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4390 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.865 | +| loss/actor | -395 | +| loss/alpha | 0.00241 | +| loss/critic | 59.1 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.6 | +| rollout_info/dynamics_penalty_mean | 0.139 | +| rollout_info/dynamics_penalty_std | 0.312 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4230 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4107 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4112 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4453 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4224 +num rollout transitions: 50000, reward mean: 4.4113 +num rollout transitions: 50000, reward mean: 4.4308 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4172 +num rollout transitions: 50000, reward mean: 4.4223 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4171 +num rollout transitions: 50000, reward mean: 4.4274 +num rollout transitions: 50000, reward mean: 4.4431 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4140 +num rollout transitions: 50000, reward mean: 4.4147 +num rollout transitions: 50000, reward mean: 4.4207 +num rollout transitions: 50000, reward mean: 4.4156 +num rollout transitions: 50000, reward mean: 4.4249 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.613 | +| loss/actor | -393 | +| loss/alpha | 0.00429 | +| loss/critic | 59 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0.14 | +| rollout_info/dynamics_penalty_std | 0.312 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4375 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4601 +num rollout transitions: 50000, reward mean: 4.4495 +num rollout transitions: 50000, reward mean: 4.4560 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4210 +num rollout transitions: 50000, reward mean: 4.4351 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4236 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4070 +num rollout transitions: 50000, reward mean: 4.4154 +num rollout transitions: 50000, reward mean: 4.4184 +num rollout transitions: 50000, reward mean: 4.4138 +num rollout transitions: 50000, reward mean: 4.4355 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4505 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4474 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4412 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4182 +num rollout transitions: 50000, reward mean: 4.4187 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4230 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 1.16 | +| loss/actor | -396 | +| loss/alpha | 0.0116 | +| loss/critic | 59.7 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.6 | +| rollout_info/dynamics_penalty_mean | 0.136 | +| rollout_info/dynamics_penalty_std | 0.308 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4206 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4165 +num rollout transitions: 50000, reward mean: 4.4354 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4161 +num rollout transitions: 50000, reward mean: 4.4073 +num rollout transitions: 50000, reward mean: 4.4162 +num rollout transitions: 50000, reward mean: 4.4180 +num rollout transitions: 50000, reward mean: 4.4183 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4078 +num rollout transitions: 50000, reward mean: 4.4074 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4314 +num rollout transitions: 50000, reward mean: 4.4103 +num rollout transitions: 50000, reward mean: 4.4162 +num rollout transitions: 50000, reward mean: 4.4106 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4459 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4482 +num rollout transitions: 50000, reward mean: 4.4414 +num rollout transitions: 50000, reward mean: 4.4218 +num rollout transitions: 50000, reward mean: 4.4072 +num rollout transitions: 50000, reward mean: 4.4190 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4245 +num rollout transitions: 50000, reward mean: 4.4008 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4145 +num rollout transitions: 50000, reward mean: 4.4437 +num rollout transitions: 50000, reward mean: 4.4313 +num rollout transitions: 50000, reward mean: 4.4259 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.127 | +| loss/actor | -398 | +| loss/alpha | -0.0163 | +| loss/critic | 60.6 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0.145 | +| rollout_info/dynamics_penalty_std | 0.317 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4455 +num rollout transitions: 50000, reward mean: 4.4501 +num rollout transitions: 50000, reward mean: 4.4425 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4104 +num rollout transitions: 50000, reward mean: 4.4132 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4129 +num rollout transitions: 50000, reward mean: 4.4179 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4232 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4427 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4539 +num rollout transitions: 50000, reward mean: 4.4276 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4514 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4259 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4479 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4119 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.142 | +| loss/actor | -394 | +| loss/alpha | -0.0117 | +| loss/critic | 60.3 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0.136 | +| rollout_info/dynamics_penalty_std | 0.309 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4184 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4121 +num rollout transitions: 50000, reward mean: 4.4345 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4311 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4295 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4151 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4219 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4481 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4312 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4041 +num rollout transitions: 50000, reward mean: 4.4141 +num rollout transitions: 50000, reward mean: 4.4149 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4454 +num rollout transitions: 50000, reward mean: 4.4405 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4525 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4253 +num rollout transitions: 50000, reward mean: 4.4229 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4029 +num rollout transitions: 50000, reward mean: 4.4212 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 994 | +| eval/episode_length_std | 60.8 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 7.39 | +| loss/actor | -395 | +| loss/alpha | 0.00996 | +| loss/critic | 60.6 | +| penalty/lcb_mean | 1.56 | +| penalty/lcb_std | 2.62 | +| rollout_info/dynamics_penalty_mean | 0.142 | +| rollout_info/dynamics_penalty_std | 0.315 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4159 +num rollout transitions: 50000, reward mean: 4.4185 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4285 +num rollout transitions: 50000, reward mean: 4.4203 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4350 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4264 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4461 +num rollout transitions: 50000, reward mean: 4.4332 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4110 +num rollout transitions: 50000, reward mean: 4.4105 +num rollout transitions: 50000, reward mean: 4.3926 +num rollout transitions: 50000, reward mean: 4.4379 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4389 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4393 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4249 +num rollout transitions: 50000, reward mean: 4.4163 +num rollout transitions: 50000, reward mean: 4.4358 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4411 +num rollout transitions: 50000, reward mean: 4.4381 +num rollout transitions: 50000, reward mean: 4.4499 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4420 +num rollout transitions: 50000, reward mean: 4.4331 +num rollout transitions: 50000, reward mean: 4.4480 +num rollout transitions: 50000, reward mean: 4.4302 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4423 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.395 | +| loss/actor | -396 | +| loss/alpha | 0.0107 | +| loss/critic | 58.3 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.58 | +| rollout_info/dynamics_penalty_mean | 0.141 | +| rollout_info/dynamics_penalty_std | 0.313 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4428 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4496 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4436 +num rollout transitions: 50000, reward mean: 4.4565 +num rollout transitions: 50000, reward mean: 4.4445 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4619 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4309 +num rollout transitions: 50000, reward mean: 4.4284 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4138 +num rollout transitions: 50000, reward mean: 4.4275 +num rollout transitions: 50000, reward mean: 4.4243 +num rollout transitions: 50000, reward mean: 4.4170 +num rollout transitions: 50000, reward mean: 4.4208 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4419 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4344 +num rollout transitions: 50000, reward mean: 4.4157 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4329 +num rollout transitions: 50000, reward mean: 4.4304 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4278 +num rollout transitions: 50000, reward mean: 4.4257 +num rollout transitions: 50000, reward mean: 4.4424 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4235 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4343 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 866 | +| eval/episode_length_std | 204 | +| eval/normalized_episode_reward | 92.4 | +| eval/normalized_episode_reward_std | 24.6 | +| loss/actor | -393 | +| loss/alpha | -0.00408 | +| loss/critic | 59.8 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.62 | +| rollout_info/dynamics_penalty_mean | 0.141 | +| rollout_info/dynamics_penalty_std | 0.314 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4472 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4519 +num rollout transitions: 50000, reward mean: 4.4460 +num rollout transitions: 50000, reward mean: 4.4522 +num rollout transitions: 50000, reward mean: 4.4578 +num rollout transitions: 50000, reward mean: 4.4415 +num rollout transitions: 50000, reward mean: 4.4408 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4489 +num rollout transitions: 50000, reward mean: 4.4293 +num rollout transitions: 50000, reward mean: 4.4353 +num rollout transitions: 50000, reward mean: 4.4282 +num rollout transitions: 50000, reward mean: 4.4252 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4227 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4624 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4197 +num rollout transitions: 50000, reward mean: 4.4273 +num rollout transitions: 50000, reward mean: 4.4426 +num rollout transitions: 50000, reward mean: 4.4528 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4328 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4271 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4390 +num rollout transitions: 50000, reward mean: 4.4410 +num rollout transitions: 50000, reward mean: 4.4439 +num rollout transitions: 50000, reward mean: 4.4384 +---------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.551 | +| loss/actor | -394 | +| loss/alpha | 0.00725 | +| loss/critic | 61.8 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.68 | +| rollout_info/dynamics_penalty_mean | 0.139 | +| rollout_info/dynamics_penalty_std | 0.312 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4357 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4395 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4346 +num rollout transitions: 50000, reward mean: 4.4442 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4446 +num rollout transitions: 50000, reward mean: 4.4363 +num rollout transitions: 50000, reward mean: 4.4456 +num rollout transitions: 50000, reward mean: 4.4337 +num rollout transitions: 50000, reward mean: 4.4407 +num rollout transitions: 50000, reward mean: 4.4380 +num rollout transitions: 50000, reward mean: 4.4478 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4438 +num rollout transitions: 50000, reward mean: 4.4488 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4241 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4398 +num rollout transitions: 50000, reward mean: 4.4435 +num rollout transitions: 50000, reward mean: 4.4432 +num rollout transitions: 50000, reward mean: 4.4462 +num rollout transitions: 50000, reward mean: 4.4269 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4250 +num rollout transitions: 50000, reward mean: 4.4200 +num rollout transitions: 50000, reward mean: 4.4268 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4261 +num rollout transitions: 50000, reward mean: 4.4330 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4277 +num rollout transitions: 50000, reward mean: 4.4429 +num rollout transitions: 50000, reward mean: 4.4287 +num rollout transitions: 50000, reward mean: 4.4272 +num rollout transitions: 50000, reward mean: 4.4198 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4277 +---------------------------------------------------------------------------------- +| alpha | 0.118 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 0.381 | +| loss/actor | -392 | +| loss/alpha | -0.00592 | +| loss/critic | 61.6 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.64 | +| rollout_info/dynamics_penalty_mean | 0.139 | +| rollout_info/dynamics_penalty_std | 0.312 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.44 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4369 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4114 +num rollout transitions: 50000, reward mean: 4.4251 +num rollout transitions: 50000, reward mean: 4.4286 +num rollout transitions: 50000, reward mean: 4.4342 +num rollout transitions: 50000, reward mean: 4.4406 +num rollout transitions: 50000, reward mean: 4.4316 +num rollout transitions: 50000, reward mean: 4.4289 +num rollout transitions: 50000, reward mean: 4.4367 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4376 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4373 +num rollout transitions: 50000, reward mean: 4.4361 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4300 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4385 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4336 +num rollout transitions: 50000, reward mean: 4.4397 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4280 +num rollout transitions: 50000, reward mean: 4.4262 +num rollout transitions: 50000, reward mean: 4.4319 +num rollout transitions: 50000, reward mean: 4.4288 +num rollout transitions: 50000, reward mean: 4.4297 +num rollout transitions: 50000, reward mean: 4.4162 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4184 +num rollout transitions: 50000, reward mean: 4.4189 +num rollout transitions: 50000, reward mean: 4.4169 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4421 +num rollout transitions: 50000, reward mean: 4.4471 +num rollout transitions: 50000, reward mean: 4.4440 +num rollout transitions: 50000, reward mean: 4.4503 +num rollout transitions: 50000, reward mean: 4.4517 +num rollout transitions: 50000, reward mean: 4.4502 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4274 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.369 | +| loss/actor | -395 | +| loss/alpha | 0.000976 | +| loss/critic | 64 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.71 | +| rollout_info/dynamics_penalty_mean | 0.145 | +| rollout_info/dynamics_penalty_std | 0.319 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4394 +num rollout transitions: 50000, reward mean: 4.4335 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.4334 +num rollout transitions: 50000, reward mean: 4.4180 +num rollout transitions: 50000, reward mean: 4.4152 +num rollout transitions: 50000, reward mean: 4.4214 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4391 +num rollout transitions: 50000, reward mean: 4.4433 +num rollout transitions: 50000, reward mean: 4.4418 +num rollout transitions: 50000, reward mean: 4.4475 +num rollout transitions: 50000, reward mean: 4.4382 +num rollout transitions: 50000, reward mean: 4.4341 +num rollout transitions: 50000, reward mean: 4.4400 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4237 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4176 +num rollout transitions: 50000, reward mean: 4.4144 +num rollout transitions: 50000, reward mean: 4.4242 +num rollout transitions: 50000, reward mean: 4.4225 +num rollout transitions: 50000, reward mean: 4.4185 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4340 +num rollout transitions: 50000, reward mean: 4.4409 +num rollout transitions: 50000, reward mean: 4.4384 +num rollout transitions: 50000, reward mean: 4.4396 +num rollout transitions: 50000, reward mean: 4.4473 +num rollout transitions: 50000, reward mean: 4.4452 +num rollout transitions: 50000, reward mean: 4.4291 +num rollout transitions: 50000, reward mean: 4.4401 +num rollout transitions: 50000, reward mean: 4.4326 +num rollout transitions: 50000, reward mean: 4.4466 +num rollout transitions: 50000, reward mean: 4.4371 +num rollout transitions: 50000, reward mean: 4.4267 +num rollout transitions: 50000, reward mean: 4.4443 +num rollout transitions: 50000, reward mean: 4.4310 +num rollout transitions: 50000, reward mean: 4.4317 +num rollout transitions: 50000, reward mean: 4.4399 +num rollout transitions: 50000, reward mean: 4.4231 +num rollout transitions: 50000, reward mean: 4.4322 +num rollout transitions: 50000, reward mean: 4.4364 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4418 +---------------------------------------------------------------------------------- +| alpha | 0.111 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 120 | +| eval/normalized_episode_reward_std | 0.742 | +| loss/actor | -393 | +| loss/alpha | -0.0115 | +| loss/critic | 62.4 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.7 | +| rollout_info/dynamics_penalty_mean | 0.145 | +| rollout_info/dynamics_penalty_std | 0.319 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4348 +num rollout transitions: 50000, reward mean: 4.4260 +num rollout transitions: 50000, reward mean: 4.4347 +num rollout transitions: 50000, reward mean: 4.4368 +num rollout transitions: 50000, reward mean: 4.4377 +num rollout transitions: 50000, reward mean: 4.4318 +num rollout transitions: 50000, reward mean: 4.4265 +num rollout transitions: 50000, reward mean: 4.4299 +num rollout transitions: 50000, reward mean: 4.4324 +num rollout transitions: 50000, reward mean: 4.4327 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4256 +num rollout transitions: 50000, reward mean: 4.4356 +num rollout transitions: 50000, reward mean: 4.4239 +num rollout transitions: 50000, reward mean: 4.4306 +num rollout transitions: 50000, reward mean: 4.4177 +num rollout transitions: 50000, reward mean: 4.4292 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4402 +num rollout transitions: 50000, reward mean: 4.4362 +num rollout transitions: 50000, reward mean: 4.4325 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4320 +num rollout transitions: 50000, reward mean: 4.4258 +num rollout transitions: 50000, reward mean: 4.4266 +num rollout transitions: 50000, reward mean: 4.4303 +num rollout transitions: 50000, reward mean: 4.4315 +num rollout transitions: 50000, reward mean: 4.4290 +num rollout transitions: 50000, reward mean: 4.4193 +num rollout transitions: 50000, reward mean: 4.4423 +num rollout transitions: 50000, reward mean: 4.4283 +num rollout transitions: 50000, reward mean: 4.4403 +num rollout transitions: 50000, reward mean: 4.4352 +num rollout transitions: 50000, reward mean: 4.4378 +num rollout transitions: 50000, reward mean: 4.4365 +num rollout transitions: 50000, reward mean: 4.4246 +num rollout transitions: 50000, reward mean: 4.4386 +num rollout transitions: 50000, reward mean: 4.4339 +num rollout transitions: 50000, reward mean: 4.4343 +num rollout transitions: 50000, reward mean: 4.4281 +num rollout transitions: 50000, reward mean: 4.4307 +num rollout transitions: 50000, reward mean: 4.4413 +num rollout transitions: 50000, reward mean: 4.4301 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4323 +num rollout transitions: 50000, reward mean: 4.4279 +num rollout transitions: 50000, reward mean: 4.4387 +num rollout transitions: 50000, reward mean: 4.4338 +num rollout transitions: 50000, reward mean: 4.4359 +num rollout transitions: 50000, reward mean: 4.4273 +---------------------------------------------------------------------------------- +| alpha | 0.0961 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 1.12 | +| loss/actor | -387 | +| loss/alpha | -0.0268 | +| loss/critic | 60.9 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0.143 | +| rollout_info/dynamics_penalty_std | 0.316 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.43 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 119925.02s diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_456×tamp_26-0319-225151/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_456×tamp_26-0319-225151/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_456×tamp_26-0319-225151/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_456×tamp_26-0319-225151/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..476b3c1767ec7d440b0fdf411be56ea424c2c34c --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_456×tamp_26-0319-225151/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_realnvp.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 456, "device": "cuda:7", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_456×tamp_26-0112-030559/model", "reward_penalty_coef": 0.5, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/realnvp", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "realnvp", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "realnvp"} diff --git a/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_456×tamp_26-0319-225151/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_456×tamp_26-0319-225151/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..0be0bc2b0b18420d1f05d708b0e8312861ced21d --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/realnvp/seed_456×tamp_26-0319-225151/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.2633093183356545,71.51,7.85938292743139,3.3241522056532578,0.3726404786193457,-281.6032238913232,-1.5959994718417623,19.384745549502682,0.9371124998951654,1.191637885024697,0.2994456012402972,0.36117690938117464,50000.0,4.375326486749285,50000 +0.16004766907960258,178.42,77.77418337726215,1.7726419727306784,5.961651689888486,-433.112473450316,0.0011745319864153265,34.39030154199546,1.2069970716345195,1.7781444073593582,0.15312713560090857,0.3243007419947268,50000.0,4.422573912752381,100000 +0.14960628921985475,683.26,244.35910541659788,62.880090720815296,24.496321000429816,-435.56227565735475,-0.008427997866924846,31.258980069980492,1.1339775537931904,1.736395300778143,0.1377972541535371,0.310798481532413,50000.0,4.445766201588416,150000 +0.1475442201709772,92.58,4.82945131459051,4.024228873604765,0.15865321065843874,-429.120532874761,0.011465840029455783,28.21285396600693,1.1089995564961317,1.5763344431269155,0.1439023423010107,0.31663975589277543,50000.0,4.444758427523191,200000 +0.15519550112992536,141.97,15.15747670293443,7.133176598892497,1.7296849087324582,-424.40134247863995,-0.00024404379639784575,31.33996620990763,1.1653243409979601,1.6847799431037862,0.13707933527467348,0.3087818289775066,50000.0,4.441883294247628,250000 +0.1602400017270454,955.51,134.3678157149248,95.00729653904425,14.102641941307304,-412.31492068786423,-0.0007119382792338383,36.535236444053375,1.2502187573134762,1.8423839094018821,0.13300981846956245,0.3055813882876794,50000.0,4.439805132930842,300000 +0.15599948044896364,977.29,101.70587937774296,96.92534423637844,10.905516327394986,-402.7034535772676,-0.0022551644584163995,37.73953063261009,1.283126163940418,1.87872626564145,0.1301229380663179,0.30264377509709917,50000.0,4.435851409050035,350000 +0.16045105598688195,779.51,280.8215979941714,77.71611562295288,29.7489501078608,-400.5894660766618,0.0033602366094663745,43.34886541582147,1.3415326026332508,2.030137091616419,0.12833466669766144,0.3005142876472858,50000.0,4.4387868938638135,400000 +0.15252058366984209,142.49,4.674387660432113,7.948083943967209,0.583036947057017,-396.00162099608923,-0.007065383525416332,42.96730930786085,1.3048925359654482,2.085383332395518,0.12602033971877843,0.29750151007679476,50000.0,4.440273395966204,450000 +0.14624011732548728,132.01,13.907188788536669,6.670699050927621,0.9820598518713595,-395.61214596679815,-0.005643351917862849,43.183870356998305,1.31131494364025,2.0962333428383206,0.12989384367669726,0.3015737085695883,50000.0,4.434890361723268,500000 +0.15106580291867247,994.55,35.76628999491001,104.71008510568076,4.5523385432591175,-394.6789102429173,0.007195917681567312,44.525738806285766,1.3463259050011673,2.1283066967570674,0.1264664542637192,0.298174038345457,50000.0,4.4386984937617635,550000 +0.1507967709761864,1000.0,0.0,109.61203153423398,0.8855498624549143,-393.4868030664058,-0.001718417072482444,47.18009266504293,1.3822652385950007,2.219780629570506,0.12861062688189268,0.30061265584918007,50000.0,4.430302919839804,600000 +0.14725648030936542,928.27,197.39588926824186,90.00515525774655,20.55668060568333,-393.1497514794911,0.006074363520815979,46.260581270160316,1.3532578051614788,2.1738033042204243,0.12623122705790957,0.29879311015188476,50000.0,4.430348043068675,650000 +0.1497139284652447,991.03,47.142222051999205,105.39244647938169,5.851114094091872,-392.47372306762503,-0.008528018851913557,49.02585658727602,1.4044323593306531,2.252679570879957,0.13015130382056658,0.3022806432417743,50000.0,4.429195638104079,700000 +0.1463046445712433,982.86,75.71882460788731,103.68054331161338,9.108269574644135,-388.4375636810309,0.0021154635839163885,47.27819078348192,1.3900087885737074,2.200955781890136,0.12232227451408133,0.294723516146505,50000.0,4.432804439625729,750000 +0.14817843159943989,994.73,37.657895586450394,109.1983235397777,4.864846692229198,-390.75399442565873,0.0022751779691873707,47.55560132692269,1.4082381744456394,2.2089996035790307,0.12578844366658726,0.29830765952140464,50000.0,4.433232068132118,800000 +0.14451985769629405,1000.0,0.0,110.17042006351296,1.343624445756043,-393.265986937258,-0.007144012828394812,46.38760145360933,1.36866645056725,2.184792027628415,0.12179221400514231,0.29428346811905265,50000.0,4.439845173233597,850000 +0.14747797101497578,1000.0,0.0,102.43580428469119,1.1157978623485614,-393.6091633880621,0.0017693895128742582,48.01332950773211,1.4022622715687694,2.22709076447845,0.12428754709409248,0.29676770426918264,50000.0,4.434511823224499,900000 +0.1463428002837308,1000.0,0.0,109.13322407825856,1.482088025621031,-392.03762186462023,0.004150966394953531,47.40982246726951,1.3960861803638838,2.2170337089645433,0.11999018766130085,0.2917740955453521,50000.0,4.439643027697546,950000 +0.14612590335458506,1000.0,0.0,108.42614952208169,0.361232897714128,-387.78448768005836,-0.001259338897354948,48.25840266345969,1.4261656679916317,2.2122601376068602,0.11837860290659087,0.29035085249997916,50000.0,4.442102076570462,1000000 +0.14503301657855366,1000.0,0.0,108.72325142708131,1.8297735228688685,-389.0010334582535,0.005884423168245779,50.33108863491077,1.447310892455567,2.2849797238981506,0.11847367919822381,0.29010339687624154,50000.0,4.43939334614725,1050000 +0.14636292946726082,1000.0,0.0,107.64443198683831,0.3271452050559094,-389.2774419433642,-0.012809289801865818,51.16895201574284,1.4630989539229886,2.29359169498447,0.1219503406789579,0.29387377113976537,50000.0,4.438879183678127,1100000 +0.14318779903471457,1000.0,0.0,109.69095441743197,0.8343094823623571,-390.8540381219457,0.003440796577930722,51.23894375017197,1.4583028428494749,2.308457933748936,0.12479325553857659,0.2964999384958307,50000.0,4.437055851447332,1150000 +0.1344443901218463,1000.0,0.0,108.09824207319095,0.6752342612406483,-393.0394822009306,0.0054375698178634425,50.528159920368466,1.4349064325261278,2.322384140070639,0.12479850199073822,0.2963342904241045,50000.0,4.435635423015346,1200000 +0.14045298773526765,1000.0,0.0,108.60299211410738,0.3834512776670333,-391.89934564086417,-0.0041034124143049105,52.11045278083775,1.4456047410941282,2.3288555328094938,0.12416873638124548,0.2958834605568778,50000.0,4.438375438321153,1250000 +0.1419604484421019,1000.0,0.0,113.5731617911584,1.3524951415048208,-393.3970174883988,0.004363715881928843,50.13323921564061,1.4474000470316324,2.293552436023897,0.125351647942712,0.2970069409388987,50000.0,4.4417084785447445,1300000 +0.14218838502943346,1000.0,0.0,104.77533728122049,1.9120284580569782,-390.5757118457077,-0.009268341084793009,52.228013723564594,1.4668515870273005,2.351968231652937,0.12694698950896563,0.2990973197298613,50000.0,4.437629615325365,1350000 +0.13846229207843325,884.8,198.9841702246689,93.51807673737598,23.08635745242123,-393.056502223513,-0.0011369701394065288,49.623112278900905,1.4357134633314732,2.299664643567795,0.12458515046351051,0.29649976195799294,50000.0,4.438875911917958,1400000 +0.14016947260201204,1000.0,0.0,110.11174733780756,0.6638463718570414,-393.9646950421137,0.002355301702804745,53.60652301975239,1.4829046192729276,2.4060648565948775,0.130596459169158,0.3028959811717266,50000.0,4.432494471828216,1450000 +0.140027488791345,990.96,60.29890877951276,109.18539610580456,7.211297394026311,-390.5643634686269,0.0036986945287881923,53.70794040187794,1.4917738229072006,2.4021238537835705,0.12544971065385796,0.29706242165438546,50000.0,4.435973773906604,1500000 +0.13763864122212002,991.44,60.90144825864161,110.99635218757777,7.384869700139271,-396.69319158630213,0.008124642373882138,55.265155492954406,1.4851548365056673,2.4712614513647324,0.13274926259957218,0.30388122521237804,50000.0,4.4324968935364675,1550000 +0.13722990612417507,1000.0,0.0,113.6751922068856,0.18789781836248667,-395.6552107605008,-0.019324443551897805,57.93184814342505,1.533331212279804,2.522612326041427,0.13309446461728514,0.30465591614133064,50000.0,4.43457425613538,1600000 +0.13762462839782386,995.17,42.1098693894911,110.2545347012423,4.952978450547583,-399.66798411926607,0.007507281996011747,59.3453436821553,1.5336474536037599,2.588573229949437,0.1369121560457189,0.30891816216176526,50000.0,4.435773000549453,1650000 +0.13770232251524567,1000.0,0.0,112.72196843788127,0.5244776421186669,-396.57582307251147,0.00144717956997456,56.02666347925225,1.5010137741994964,2.5108137854791077,0.12913997342003597,0.30052567072095454,50000.0,4.441354033698712,1700000 +0.14072435047388157,1000.0,0.0,111.96990884189705,1.3183663950235485,-396.0981269787618,-8.62130027269297e-06,56.045122129115995,1.5181664439511415,2.4905384993529274,0.13331373355067483,0.3048571846296456,50000.0,4.437875075152837,1750000 +0.1401973596346374,1000.0,0.0,113.68877296363706,0.5921906751780535,-393.64905294982583,0.004319016932211785,56.450521305389856,1.5383068576908092,2.501145171325212,0.13043755208723776,0.30156413241432195,50000.0,4.438975736475202,1800000 +0.13907186192512594,1000.0,0.0,115.65546768366907,0.44608637701118403,-397.17869240539267,-0.003871040758192595,59.00733875326155,1.5498493392562787,2.5655009670734494,0.13420445267321965,0.3055822835082252,50000.0,4.4399576945655745,1850000 +0.1442179736211882,1000.0,0.0,112.6337040233032,0.20676577148228972,-395.9108905987517,0.007790288354456482,61.053592056178935,1.5951824268734296,2.603740655512805,0.13551845998580767,0.30659831277308347,50000.0,4.431819933309234,1900000 +0.14149158484697316,1000.0,0.0,115.49350637814257,0.33520837823174277,-398.6301102813759,0.003311462814137417,61.92053957397442,1.620002320713992,2.6273208004367024,0.14212850191829238,0.31408350745736946,50000.0,4.434367737571889,1950000 +0.13771680096313157,1000.0,0.0,113.85950573565668,1.0249433814947875,-398.9835642376713,-0.011454368946142508,62.22585380706814,1.5902973326969367,2.6343965497804254,0.14132497779037384,0.31320689408450425,50000.0,4.4365498845155225,2000000 +0.13429502898991053,1000.0,0.0,115.33237826156741,0.3513775519747819,-398.22902744933737,-0.0020279914120026484,58.86476319892889,1.5478516054415847,2.588481823241712,0.13925939833798506,0.31168172562112795,50000.0,4.435891430468171,2050000 +0.13500869324505094,925.05,129.9851049159095,104.00358185807279,16.10263642372321,-399.37718161498987,-0.003020957709178201,59.23661109949115,1.545604269649992,2.588803359642032,0.1340229885211909,0.3059596987521308,50000.0,4.440332528202131,2100000 +0.13052869388118324,1000.0,0.0,113.28035525611494,0.21611327452308632,-398.45517398498777,0.012665743186920932,58.127119775734606,1.5482592117833969,2.586205138099164,0.14014848055411447,0.3120727992001551,50000.0,4.435428788132564,2150000 +0.12493056726813213,997.8,21.889723616345652,111.25156077356073,3.149512701468858,-400.2285727648938,-0.010071944151446188,56.92654824220665,1.5214192372858577,2.555927995088107,0.1485353104563883,0.32114981515597135,50000.0,4.428749469811711,2200000 +0.12391229466795843,1000.0,0.0,116.4031895288435,0.6234373005637184,-398.64463412719726,0.006933373796269297,59.613803024673835,1.5481622963619195,2.6261304086398907,0.1433483182149286,0.31578358760335357,50000.0,4.4286368179371856,2250000 +0.12949406926736368,1000.0,0.0,114.80059118143078,0.2517969705264016,-397.3469940283248,0.00309115279156724,59.24751135402669,1.5483872022199692,2.6314838626503976,0.14006815361824987,0.3126581033945519,50000.0,4.427772536721629,2300000 +0.11920593940481612,1000.0,0.0,112.61063139210495,0.22576614136475023,-399.41521015990907,-0.012212708431631447,56.72030986913705,1.5040859657609296,2.56657556957124,0.14161959340848995,0.31404602235017137,50000.0,4.428621040434277,2350000 +0.1267145266240814,1000.0,0.0,113.25253127342718,0.864524575829983,-395.3293761297588,0.002405024402514075,59.14741898506187,1.545544047040944,2.5964245789956877,0.13948196957149478,0.3124999818504062,50000.0,4.434951020576156,2400000 +0.12766656539082533,1000.0,0.0,113.71160613584499,0.6133855088363658,-392.91254052978576,0.004285599000565728,59.00146284816778,1.54863473451138,2.592000273277754,0.13950159874683918,0.31209198618964834,50000.0,4.430705734341853,2450000 +0.12703375876516343,1000.0,0.0,111.6340735132925,1.1555308845278696,-396.1661699768012,0.011574207448214302,59.74865471616684,1.551659613089569,2.601433358547704,0.13615723065599097,0.3082379730939231,50000.0,4.433904106884436,2500000 +0.12460174640655615,1000.0,0.0,117.44638903803184,0.12703518791730373,-397.9969884735072,-0.016337315928898335,60.56391694560982,1.554541115319722,2.6535090338849976,0.14533349885181374,0.3173905837633863,50000.0,4.425445700758943,2550000 +0.12070927097514217,1000.0,0.0,117.39458057510606,0.14198098137776677,-394.4719300793403,-0.011715064972862772,60.27158938960976,1.5331215763008559,2.6451878439879297,0.13628701182319572,0.3089384913512267,50000.0,4.433123297965856,2600000 +0.12032313403025321,993.89,60.79373240721448,111.13849090309633,7.386540232873831,-394.97742868652216,0.009961179746314873,60.59232199602133,1.5623533738231778,2.6246294707107363,0.14227056370251423,0.3151175603963763,50000.0,4.429258399936488,2650000 +0.11897514724552362,1000.0,0.0,114.8673761636271,0.3952237156679443,-396.0091022515864,0.010662507581785288,58.32656188463271,1.5505864419102742,2.582690831309539,0.14060512777013331,0.313387401015078,50000.0,4.431861271333587,2700000 +0.12019142254516417,865.89,204.2479813853738,92.40898250268906,24.60352815489419,-393.0035666534369,-0.0040801542052624025,59.82033004182851,1.5754784534192035,2.616992225713686,0.14114683589440044,0.314232857942433,50000.0,4.4345348376125155,2750000 +0.12191792370796264,1000.0,0.0,117.37073658986104,0.5511928537579059,-393.75462694152736,0.007245555966049523,61.80509336843519,1.6018595802664766,2.6782298286724235,0.13869661791991333,0.31212028490402843,50000.0,4.436469766964168,2800000 +0.1182783201560373,1000.0,0.0,118.29001017429952,0.38109392922268026,-392.24865036498784,-0.005917212074510526,61.556114192790965,1.597226257798656,2.6411552940798027,0.13884331483428494,0.31198148594222586,50000.0,4.435075037093532,2850000 +0.11969865155890566,1000.0,0.0,113.92451605349439,0.3694359309449646,-394.76966849975446,0.0009764044121650963,64.00026304380366,1.6306410957646345,2.7147952618670277,0.14544155264894254,0.31887992720366104,50000.0,4.433150046736802,2900000 +0.11102862871959651,1000.0,0.0,119.86474630389641,0.7418895342851919,-392.9177860125737,-0.01146065501771867,62.39560249710138,1.593263587102875,2.6952750716805847,0.1452060638278887,0.3186800629643108,50000.0,4.433658090315141,2950000 +0.09612781278520895,1000.0,0.0,118.17251014429921,1.1151396249192986,-386.8854606768828,-0.02679252565808627,60.91959548419971,1.5906396827387688,2.6491241148662885,0.1427873089099688,0.3162443417775155,50000.0,4.431968891470503,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-171722/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-171722/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..9fb80affb754978526062ba5ba372445fbd980f6 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-171722/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 4.4372 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.5628 +num rollout transitions: 50000, reward mean: 4.5397 +num rollout transitions: 50000, reward mean: 4.5290 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5270 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5278 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4732 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5365 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5314 +num rollout transitions: 50000, reward mean: 4.5305 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4807 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.3359 +num rollout transitions: 50000, reward mean: 3.9892 +num rollout transitions: 50000, reward mean: 4.2141 +num rollout transitions: 50000, reward mean: 4.2449 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5245 +num rollout transitions: 50000, reward mean: 4.5283 +num rollout transitions: 50000, reward mean: 4.5346 +num rollout transitions: 50000, reward mean: 4.5308 +num rollout transitions: 50000, reward mean: 4.5377 +num rollout transitions: 50000, reward mean: 4.5436 +num rollout transitions: 50000, reward mean: 4.5215 +---------------------------------------------------------------------------------- +| alpha | 0.302 | +| eval/episode_length | 51.7 | +| eval/episode_length_std | 0.826 | +| eval/normalized_episode_reward | -0.305 | +| eval/normalized_episode_reward_std | -0.0182 | +| loss/actor | -395 | +| loss/alpha | -0.998 | +| loss/critic | 62.1 | +| penalty/lcb_mean | 1.8 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 0.00574 | +| rollout_info/dynamics_penalty_std | 0.0451 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5356 +num rollout transitions: 50000, reward mean: 4.5379 +num rollout transitions: 50000, reward mean: 4.5286 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5273 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5240 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5255 +num rollout transitions: 50000, reward mean: 4.5401 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5271 +num rollout transitions: 50000, reward mean: 4.4759 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5272 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5021 +---------------------------------------------------------------------------------- +| alpha | 0.171 | +| eval/episode_length | 164 | +| eval/episode_length_std | 33.4 | +| eval/normalized_episode_reward | 13 | +| eval/normalized_episode_reward_std | 4.21 | +| loss/actor | -524 | +| loss/alpha | -0.0424 | +| loss/critic | 41.4 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 1.97 | +| rollout_info/dynamics_penalty_mean | 0.00112 | +| rollout_info/dynamics_penalty_std | 0.0273 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5245 +num rollout transitions: 50000, reward mean: 4.5409 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5398 +num rollout transitions: 50000, reward mean: 4.5295 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5267 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5299 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5281 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.5432 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5384 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5332 +num rollout transitions: 50000, reward mean: 4.5307 +num rollout transitions: 50000, reward mean: 4.5328 +num rollout transitions: 50000, reward mean: 4.5294 +num rollout transitions: 50000, reward mean: 4.5388 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.5277 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5469 +num rollout transitions: 50000, reward mean: 4.5279 +num rollout transitions: 50000, reward mean: 4.5309 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.5372 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5189 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 352 | +| eval/episode_length_std | 95.3 | +| eval/normalized_episode_reward | 29.7 | +| eval/normalized_episode_reward_std | 11.5 | +| loss/actor | -453 | +| loss/alpha | -0.00641 | +| loss/critic | 33.8 | +| penalty/lcb_mean | 1.21 | +| penalty/lcb_std | 1.79 | +| rollout_info/dynamics_penalty_mean | 0.00126 | +| rollout_info/dynamics_penalty_std | 0.0295 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.52 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5274 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5301 +num rollout transitions: 50000, reward mean: 4.5351 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5234 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5317 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5299 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5058 +---------------------------------------------------------------------------------- +| alpha | 0.154 | +| eval/episode_length | 104 | +| eval/episode_length_std | 4.56 | +| eval/normalized_episode_reward | 4.56 | +| eval/normalized_episode_reward_std | 0.179 | +| loss/actor | -435 | +| loss/alpha | 0.0118 | +| loss/critic | 36.5 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 1.81 | +| rollout_info/dynamics_penalty_mean | 0.001 | +| rollout_info/dynamics_penalty_std | 0.0256 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5326 +num rollout transitions: 50000, reward mean: 4.5271 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5336 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5323 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5272 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5118 +---------------------------------------------------------------------------------- +| alpha | 0.158 | +| eval/episode_length | 553 | +| eval/episode_length_std | 326 | +| eval/normalized_episode_reward | 52.2 | +| eval/normalized_episode_reward_std | 35.2 | +| loss/actor | -421 | +| loss/alpha | -0.00512 | +| loss/critic | 39.1 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 1.84 | +| rollout_info/dynamics_penalty_mean | 0.00087 | +| rollout_info/dynamics_penalty_std | 0.0237 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4713 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4784 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4837 +num rollout transitions: 50000, reward mean: 4.4807 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4785 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4845 +---------------------------------------------------------------------------------- +| alpha | 0.155 | +| eval/episode_length | 124 | +| eval/episode_length_std | 2.32 | +| eval/normalized_episode_reward | 6.48 | +| eval/normalized_episode_reward_std | 0.128 | +| loss/actor | -409 | +| loss/alpha | -0.00492 | +| loss/critic | 35.6 | +| penalty/lcb_mean | 1.24 | +| penalty/lcb_std | 1.77 | +| rollout_info/dynamics_penalty_mean | 0.000794 | +| rollout_info/dynamics_penalty_std | 0.0225 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.4679 +num rollout transitions: 50000, reward mean: 4.4680 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4762 +num rollout transitions: 50000, reward mean: 4.4789 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4766 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.4734 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.4807 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4923 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 693 | +| eval/episode_length_std | 241 | +| eval/normalized_episode_reward | 71.6 | +| eval/normalized_episode_reward_std | 28 | +| loss/actor | -411 | +| loss/alpha | -0.00418 | +| loss/critic | 36.6 | +| penalty/lcb_mean | 1.25 | +| penalty/lcb_std | 1.81 | +| rollout_info/dynamics_penalty_mean | 0.000751 | +| rollout_info/dynamics_penalty_std | 0.0218 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4840 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4716 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4774 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.4672 +num rollout transitions: 50000, reward mean: 4.4782 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4717 +num rollout transitions: 50000, reward mean: 4.5115 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 147 | +| eval/episode_length_std | 1.11 | +| eval/normalized_episode_reward | 9.55 | +| eval/normalized_episode_reward_std | 0.0993 | +| loss/actor | -413 | +| loss/alpha | 0.000334 | +| loss/critic | 38.5 | +| penalty/lcb_mean | 1.28 | +| penalty/lcb_std | 1.87 | +| rollout_info/dynamics_penalty_mean | 0.000708 | +| rollout_info/dynamics_penalty_std | 0.0212 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4639 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.4785 +num rollout transitions: 50000, reward mean: 4.4715 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5290 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4896 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 950 | +| eval/episode_length_std | 122 | +| eval/normalized_episode_reward | 100 | +| eval/normalized_episode_reward_std | 13.8 | +| loss/actor | -413 | +| loss/alpha | -0.00777 | +| loss/critic | 37.6 | +| penalty/lcb_mean | 1.24 | +| penalty/lcb_std | 1.86 | +| rollout_info/dynamics_penalty_mean | 0.000681 | +| rollout_info/dynamics_penalty_std | 0.0206 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4798 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.4766 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5222 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5098 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 963 | +| eval/episode_length_std | 135 | +| eval/normalized_episode_reward | 100 | +| eval/normalized_episode_reward_std | 15.2 | +| loss/actor | -414 | +| loss/alpha | -0.00823 | +| loss/critic | 35 | +| penalty/lcb_mean | 1.18 | +| penalty/lcb_std | 1.81 | +| rollout_info/dynamics_penalty_mean | 0.000715 | +| rollout_info/dynamics_penalty_std | 0.0211 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4721 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.5264 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4779 +num rollout transitions: 50000, reward mean: 4.4775 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4744 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5068 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 987 | +| eval/episode_length_std | 64.2 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 7.34 | +| loss/actor | -416 | +| loss/alpha | 0.00949 | +| loss/critic | 34.7 | +| penalty/lcb_mean | 1.17 | +| penalty/lcb_std | 1.81 | +| rollout_info/dynamics_penalty_mean | 0.000698 | +| rollout_info/dynamics_penalty_std | 0.0209 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5273 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5252 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5187 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.872 | +| loss/actor | -412 | +| loss/alpha | -0.0112 | +| loss/critic | 36 | +| penalty/lcb_mean | 1.19 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 0.000625 | +| rollout_info/dynamics_penalty_std | 0.0197 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4796 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4713 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5293 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4840 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4723 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.4983 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 0.254 | +| loss/actor | -412 | +| loss/alpha | 0.0012 | +| loss/critic | 34.9 | +| penalty/lcb_mean | 1.16 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 0.000586 | +| rollout_info/dynamics_penalty_std | 0.019 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4812 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4832 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5292 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4805 +num rollout transitions: 50000, reward mean: 4.4798 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5264 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.4930 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.534 | +| loss/actor | -411 | +| loss/alpha | 0.0125 | +| loss/critic | 38.3 | +| penalty/lcb_mean | 1.21 | +| penalty/lcb_std | 1.98 | +| rollout_info/dynamics_penalty_mean | 0.000639 | +| rollout_info/dynamics_penalty_std | 0.02 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.5253 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5256 +num rollout transitions: 50000, reward mean: 4.4800 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 940 | +| eval/episode_length_std | 150 | +| eval/normalized_episode_reward | 99.8 | +| eval/normalized_episode_reward_std | 17.5 | +| loss/actor | -411 | +| loss/alpha | -0.00919 | +| loss/critic | 38.7 | +| penalty/lcb_mean | 1.23 | +| penalty/lcb_std | 1.99 | +| rollout_info/dynamics_penalty_mean | 0.000615 | +| rollout_info/dynamics_penalty_std | 0.0195 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4833 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4726 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4725 +num rollout transitions: 50000, reward mean: 4.4780 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4795 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4776 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.4970 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 992 | +| eval/episode_length_std | 48.5 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 5.74 | +| loss/actor | -410 | +| loss/alpha | 0.00509 | +| loss/critic | 38.7 | +| penalty/lcb_mean | 1.23 | +| penalty/lcb_std | 2.01 | +| rollout_info/dynamics_penalty_mean | 0.000638 | +| rollout_info/dynamics_penalty_std | 0.0201 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4750 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4772 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5282 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5265 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4894 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 0.356 | +| loss/actor | -409 | +| loss/alpha | 0.00758 | +| loss/critic | 40.4 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 2.04 | +| rollout_info/dynamics_penalty_mean | 0.000587 | +| rollout_info/dynamics_penalty_std | 0.0191 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4724 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4731 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4726 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4776 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4819 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4895 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.7 | +| eval/normalized_episode_reward_std | 0.723 | +| loss/actor | -411 | +| loss/alpha | -0.00942 | +| loss/critic | 40.9 | +| penalty/lcb_mean | 1.28 | +| penalty/lcb_std | 2.05 | +| rollout_info/dynamics_penalty_mean | 0.00059 | +| rollout_info/dynamics_penalty_std | 0.0192 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4740 +num rollout transitions: 50000, reward mean: 4.5266 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4801 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4610 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5205 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5176 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 1.02 | +| loss/actor | -413 | +| loss/alpha | 0.00111 | +| loss/critic | 42 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 2.1 | +| rollout_info/dynamics_penalty_mean | 0.000629 | +| rollout_info/dynamics_penalty_std | 0.0196 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4835 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5253 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4748 +num rollout transitions: 50000, reward mean: 4.4762 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.4905 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 975 | +| eval/episode_length_std | 90.5 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 10.8 | +| loss/actor | -412 | +| loss/alpha | 0.00175 | +| loss/critic | 41.9 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 2.08 | +| rollout_info/dynamics_penalty_mean | 0.00061 | +| rollout_info/dynamics_penalty_std | 0.0194 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4794 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.4723 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4782 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4857 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 957 | +| eval/episode_length_std | 119 | +| eval/normalized_episode_reward | 100 | +| eval/normalized_episode_reward_std | 15.7 | +| loss/actor | -409 | +| loss/alpha | 0.00117 | +| loss/critic | 42.7 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.11 | +| rollout_info/dynamics_penalty_mean | 0.000634 | +| rollout_info/dynamics_penalty_std | 0.02 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5289 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5067 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 1.01 | +| loss/actor | -411 | +| loss/alpha | -0.00498 | +| loss/critic | 43 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.1 | +| rollout_info/dynamics_penalty_mean | 0.000552 | +| rollout_info/dynamics_penalty_std | 0.0184 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4716 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4794 +num rollout transitions: 50000, reward mean: 4.4824 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4991 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 968 | +| eval/episode_length_std | 122 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 14.7 | +| loss/actor | -412 | +| loss/alpha | 0.00828 | +| loss/critic | 41.9 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 2.12 | +| rollout_info/dynamics_penalty_mean | 0.000671 | +| rollout_info/dynamics_penalty_std | 0.0207 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4786 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5272 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.4783 +num rollout transitions: 50000, reward mean: 4.4819 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4842 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 910 | +| eval/episode_length_std | 178 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 21.9 | +| loss/actor | -416 | +| loss/alpha | -0.00906 | +| loss/critic | 42.8 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.16 | +| rollout_info/dynamics_penalty_mean | 0.000645 | +| rollout_info/dynamics_penalty_std | 0.0202 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4833 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5043 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 882 | +| eval/episode_length_std | 194 | +| eval/normalized_episode_reward | 96.5 | +| eval/normalized_episode_reward_std | 23 | +| loss/actor | -415 | +| loss/alpha | 0.0036 | +| loss/critic | 44 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.18 | +| rollout_info/dynamics_penalty_mean | 0.000702 | +| rollout_info/dynamics_penalty_std | 0.021 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.4667 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5049 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 986 | +| eval/episode_length_std | 62.6 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 7.72 | +| loss/actor | -417 | +| loss/alpha | -0.0045 | +| loss/critic | 43.8 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.18 | +| rollout_info/dynamics_penalty_mean | 0.000649 | +| rollout_info/dynamics_penalty_std | 0.0202 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4990 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 985 | +| eval/episode_length_std | 54.5 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 7.66 | +| loss/actor | -415 | +| loss/alpha | 0.00977 | +| loss/critic | 45.2 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.25 | +| rollout_info/dynamics_penalty_mean | 0.00062 | +| rollout_info/dynamics_penalty_std | 0.0198 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4837 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5042 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 911 | +| eval/episode_length_std | 167 | +| eval/normalized_episode_reward | 98.3 | +| eval/normalized_episode_reward_std | 20 | +| loss/actor | -411 | +| loss/alpha | -0.00266 | +| loss/critic | 43.8 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 0.00056 | +| rollout_info/dynamics_penalty_std | 0.0185 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5281 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4862 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 850 | +| eval/episode_length_std | 174 | +| eval/normalized_episode_reward | 86.5 | +| eval/normalized_episode_reward_std | 20.1 | +| loss/actor | -412 | +| loss/alpha | -0.00259 | +| loss/critic | 45.5 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 0.000539 | +| rollout_info/dynamics_penalty_std | 0.0181 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4794 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5293 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4815 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.5043 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 619 | +| eval/episode_length_std | 219 | +| eval/normalized_episode_reward | 62.9 | +| eval/normalized_episode_reward_std | 25.9 | +| loss/actor | -413 | +| loss/alpha | 0.00303 | +| loss/critic | 44.9 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 0.000612 | +| rollout_info/dynamics_penalty_std | 0.0193 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5273 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4705 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5199 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 930 | +| eval/episode_length_std | 146 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 17.2 | +| loss/actor | -410 | +| loss/alpha | -0.00577 | +| loss/critic | 46 | +| penalty/lcb_mean | 1.34 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 0.000567 | +| rollout_info/dynamics_penalty_std | 0.0186 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5251 +num rollout transitions: 50000, reward mean: 4.5290 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4710 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.4797 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5328 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5280 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5317 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5030 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 999 | +| eval/episode_length_std | 5.89 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.15 | +| loss/actor | -410 | +| loss/alpha | -0.00405 | +| loss/critic | 44.6 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 0.000594 | +| rollout_info/dynamics_penalty_std | 0.0192 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5261 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5086 +---------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 984 | +| eval/episode_length_std | 64 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 7.36 | +| loss/actor | -412 | +| loss/alpha | -0.00419 | +| loss/critic | 44.8 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 0.000619 | +| rollout_info/dynamics_penalty_std | 0.0198 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4785 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5312 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.4908 +----------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.27 | +| loss/actor | -413 | +| loss/alpha | -0.000361 | +| loss/critic | 45.5 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.22 | +| rollout_info/dynamics_penalty_mean | 0.000611 | +| rollout_info/dynamics_penalty_std | 0.0196 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1700000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5265 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4796 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5272 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5122 +---------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.31 | +| loss/actor | -415 | +| loss/alpha | -0.00112 | +| loss/critic | 45.9 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.26 | +| rollout_info/dynamics_penalty_mean | 0.000576 | +| rollout_info/dynamics_penalty_std | 0.0189 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5278 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4820 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5019 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 4.18 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.22 | +| loss/actor | -417 | +| loss/alpha | 0.00652 | +| loss/critic | 45.1 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 0.000545 | +| rollout_info/dynamics_penalty_std | 0.0185 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5318 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5070 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 998 | +| eval/episode_length_std | 17.4 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 2.85 | +| loss/actor | -417 | +| loss/alpha | -0.0169 | +| loss/critic | 44 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 2.23 | +| rollout_info/dynamics_penalty_mean | 0.000568 | +| rollout_info/dynamics_penalty_std | 0.0191 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5331 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5056 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 1.84 | +| loss/actor | -408 | +| loss/alpha | 0.00882 | +| loss/critic | 47.9 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.29 | +| rollout_info/dynamics_penalty_mean | 0.000567 | +| rollout_info/dynamics_penalty_std | 0.0188 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5366 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4811 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4975 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 1.92 | +| loss/actor | -414 | +| loss/alpha | 0.0108 | +| loss/critic | 46.3 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 0.000578 | +| rollout_info/dynamics_penalty_std | 0.019 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5258 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5021 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.298 | +| loss/actor | -412 | +| loss/alpha | -0.00736 | +| loss/critic | 45.5 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.25 | +| rollout_info/dynamics_penalty_mean | 0.000551 | +| rollout_info/dynamics_penalty_std | 0.0185 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5264 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5007 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.44 | +| loss/actor | -412 | +| loss/alpha | 0.0116 | +| loss/critic | 50.8 | +| penalty/lcb_mean | 1.43 | +| penalty/lcb_std | 2.37 | +| rollout_info/dynamics_penalty_mean | 0.000526 | +| rollout_info/dynamics_penalty_std | 0.018 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4833 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4820 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5249 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5083 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.899 | +| loss/actor | -413 | +| loss/alpha | -0.00395 | +| loss/critic | 48.5 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 0.000531 | +| rollout_info/dynamics_penalty_std | 0.0181 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5223 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4847 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4884 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 904 | +| eval/episode_length_std | 180 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 22.9 | +| loss/actor | -416 | +| loss/alpha | -0.00359 | +| loss/critic | 47.8 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.3 | +| rollout_info/dynamics_penalty_mean | 0.000514 | +| rollout_info/dynamics_penalty_std | 0.0177 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5267 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4984 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.589 | +| loss/actor | -418 | +| loss/alpha | -0.0131 | +| loss/critic | 46.7 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.29 | +| rollout_info/dynamics_penalty_mean | 0.000549 | +| rollout_info/dynamics_penalty_std | 0.0185 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4786 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5219 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4840 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.4930 +---------------------------------------------------------------------------------- +| alpha | 0.115 | +| eval/episode_length | 988 | +| eval/episode_length_std | 58.9 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 7.28 | +| loss/actor | -419 | +| loss/alpha | 0.00339 | +| loss/critic | 45.9 | +| penalty/lcb_mean | 1.36 | +| penalty/lcb_std | 2.27 | +| rollout_info/dynamics_penalty_mean | 0.000542 | +| rollout_info/dynamics_penalty_std | 0.0183 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5234 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4787 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5052 +---------------------------------------------------------------------------------- +| alpha | 0.115 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.1 | +| loss/actor | -417 | +| loss/alpha | 0.00287 | +| loss/critic | 45.3 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.26 | +| rollout_info/dynamics_penalty_mean | 0.000544 | +| rollout_info/dynamics_penalty_std | 0.0184 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5314 +num rollout transitions: 50000, reward mean: 4.5355 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4789 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.5311 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5019 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.7 | +| loss/actor | -417 | +| loss/alpha | 0.0102 | +| loss/critic | 48.7 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 0.000591 | +| rollout_info/dynamics_penalty_std | 0.0192 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4833 +num rollout transitions: 50000, reward mean: 4.4781 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5058 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 994 | +| eval/episode_length_std | 37.2 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 4.41 | +| loss/actor | -411 | +| loss/alpha | -0.00571 | +| loss/critic | 49.6 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.000524 | +| rollout_info/dynamics_penalty_std | 0.0181 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5001 +---------------------------------------------------------------------------------- +| alpha | 0.114 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.676 | +| loss/actor | -415 | +| loss/alpha | 0.000258 | +| loss/critic | 50.6 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 0.000543 | +| rollout_info/dynamics_penalty_std | 0.0184 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5031 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 976 | +| eval/episode_length_std | 80 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 9.92 | +| loss/actor | -416 | +| loss/alpha | 0.000904 | +| loss/critic | 52.4 | +| penalty/lcb_mean | 1.45 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0.00051 | +| rollout_info/dynamics_penalty_std | 0.0177 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5249 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5263 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5063 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 1.99 | +| eval/normalized_episode_reward | 119 | +| eval/normalized_episode_reward_std | 0.998 | +| loss/actor | -414 | +| loss/alpha | 0.0033 | +| loss/critic | 50 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 0.00055 | +| rollout_info/dynamics_penalty_std | 0.0186 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4891 +---------------------------------------------------------------------------------- +| alpha | 0.109 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.383 | +| loss/actor | -413 | +| loss/alpha | -0.0139 | +| loss/critic | 48.2 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.000519 | +| rollout_info/dynamics_penalty_std | 0.0178 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4896 +---------------------------------------------------------------------------------- +| alpha | 0.109 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.766 | +| loss/actor | -414 | +| loss/alpha | 0.0114 | +| loss/critic | 49 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 0.000524 | +| rollout_info/dynamics_penalty_std | 0.0179 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5067 +---------------------------------------------------------------------------------- +| alpha | 0.113 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 0.568 | +| loss/actor | -416 | +| loss/alpha | 0.019 | +| loss/critic | 49.3 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.000529 | +| rollout_info/dynamics_penalty_std | 0.018 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5268 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5283 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4803 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5056 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.603 | +| loss/actor | -409 | +| loss/alpha | -0.0175 | +| loss/critic | 53.4 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.44 | +| rollout_info/dynamics_penalty_mean | 0.00052 | +| rollout_info/dynamics_penalty_std | 0.018 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5217 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5226 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5313 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5136 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.772 | +| loss/actor | -409 | +| loss/alpha | 0.000205 | +| loss/critic | 55.5 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.47 | +| rollout_info/dynamics_penalty_mean | 0.000512 | +| rollout_info/dynamics_penalty_std | 0.0178 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5014 +---------------------------------------------------------------------------------- +| alpha | 0.115 | +| eval/episode_length | 982 | +| eval/episode_length_std | 67.6 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 8.46 | +| loss/actor | -411 | +| loss/alpha | -0.00235 | +| loss/critic | 55.4 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.47 | +| rollout_info/dynamics_penalty_mean | 0.000535 | +| rollout_info/dynamics_penalty_std | 0.0181 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.5152 +---------------------------------------------------------------------------------- +| alpha | 0.105 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.335 | +| loss/actor | -412 | +| loss/alpha | -0.0236 | +| loss/critic | 55.6 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.49 | +| rollout_info/dynamics_penalty_mean | 0.000519 | +| rollout_info/dynamics_penalty_std | 0.0179 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5028 +---------------------------------------------------------------------------------- +| alpha | 0.106 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 117 | +| eval/normalized_episode_reward_std | 0.35 | +| loss/actor | -408 | +| loss/alpha | 0.000404 | +| loss/critic | 52.2 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0.000516 | +| rollout_info/dynamics_penalty_std | 0.0178 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5061 +---------------------------------------------------------------------------------- +| alpha | 0.108 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 0.323 | +| loss/actor | -399 | +| loss/alpha | -0.0249 | +| loss/critic | 53.4 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.43 | +| rollout_info/dynamics_penalty_mean | 0.000519 | +| rollout_info/dynamics_penalty_std | 0.018 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 128763.66s diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-171722/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-171722/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-171722/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-171722/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..105ddd33c81ba126a3546059904ffc9a62fa0033 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-171722/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_vae.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 123, "device": "cuda:4", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_123×tamp_26-0112-030558/model", "reward_penalty_coef": 0.7, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/vae", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "vae", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "vae"} diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-171722/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-171722/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..764ec71e8d1fbe17e0ce0b032e70cb92105424e8 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/vae/seed_123×tamp_26-0318-171722/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.30209859608650164,51.72,0.8255906976220118,-0.3049553837354781,-0.01820941349487064,-394.56845704106325,-0.9978321505881117,62.0500234610281,1.7952706861375094,2.312749878340953,0.005735496870310717,0.045074760069925376,50000.0,4.487362319651776,50000 +0.17051938573271017,164.17,33.438915951328326,12.963227434992689,4.212185548119223,-523.7450470452951,-0.042435037753320935,41.35867185424849,1.3197112905168444,1.9733475257718978,0.0011174817983329083,0.02734047192579738,50000.0,4.510777949481644,100000 +0.14461665858506934,351.53,95.25402406197861,29.67018839057401,11.543328216048689,-453.15192794006174,-0.006405823499486129,33.81438779115635,1.211063602260339,1.7883266236674757,0.0012589915060456994,0.029530321424926335,50000.0,4.521233567539117,150000 +0.15442323778390518,103.71,4.559155623577682,4.556328144460545,0.17903629634177262,-435.24274726501244,0.011826514859385958,36.48648422346093,1.2615884800040933,1.8086024427342282,0.0010028570711313224,0.025619241845278865,50000.0,4.5109039130570245,200000 +0.15761841701150056,553.06,325.7753158236517,52.19914425452438,35.21707775420511,-421.12787021789654,-0.005121918110996239,39.13971348258953,1.298037015662203,1.8386728121793416,0.00087010867401181,0.02369135180872365,50000.0,4.513479990900478,250000 +0.1548150710442656,123.53,2.3171318477807863,6.479389915720708,0.12796800319109403,-408.9877777697742,-0.004923010163009127,35.56233126594494,1.237207778201108,1.7728879781472622,0.0007940773402361434,0.02247469311463683,50000.0,4.500060110307178,300000 +0.14508834345907068,692.96,240.70953117814008,71.55480608299195,27.983635249403026,-411.1494437335158,-0.004183575695641376,36.642775481090446,1.253462081706517,1.8131079282414702,0.0007513777172543096,0.02178581113502293,50000.0,4.495340694207187,350000 +0.14458134785413682,146.6,1.1135528725660042,9.545717229027966,0.09933405913570652,-413.05414075439734,0.0003340564346313329,38.48456168102327,1.2766813073432406,1.870029904447782,0.0007084370728586064,0.021224064749120336,50000.0,4.497578928852719,400000 +0.1378441723689475,949.77,121.67784145028214,100.1122722613105,13.828844866326145,-412.5270523394743,-0.0077681464354319415,37.60056153415598,1.2436544819593542,1.8581689213383092,0.0006809035534707598,0.020607522165990988,50000.0,4.498961375948697,450000 +0.131788399921506,963.32,134.72281766649627,100.49725500762734,15.165363333035115,-413.9162324731488,-0.008232025422454057,35.020846476231085,1.1801778896010033,1.806593345776785,0.0007147249118238271,0.021145142233921343,50000.0,4.498935108530366,500000 +0.12884010196551302,986.73,64.17925755257691,103.11268197804628,7.342936624017732,-415.5821260754424,0.00948901083670553,34.685838199786986,1.1740308218026234,1.8052810347533141,0.000698233669247737,0.020918356051333,50000.0,4.500687592583799,550000 +0.12642335153356454,1000.0,0.0,109.47177353592991,0.8719295001519788,-412.3540140838631,-0.011184902591854414,35.98224223304797,1.1945908177399833,1.8816269828987287,0.0006247179701909526,0.019743038865200322,50000.0,4.503186619543353,600000 +0.12033730342343406,1000.0,0.0,106.91150695212474,0.25412705498817045,-412.23124589416915,0.001204188465215243,34.87037289567971,1.1629106972169767,1.8800652867353091,0.0005855564758565625,0.019010029748655045,50000.0,4.500344029410018,650000 +0.12277718115910931,1000.0,0.0,108.5451227831547,0.5343889269304393,-410.6559284698478,0.012523505658507314,38.32116393770227,1.2096686218691073,1.9790910753309736,0.0006394279139856047,0.020002454639260946,50000.0,4.502212113937062,700000 +0.131726790208969,940.06,149.64008954822236,99.78981907220921,17.46155226441533,-411.18576338501646,-0.009192742706350886,38.743817042369905,1.2302152650893006,1.9949014302993116,0.0006145476088538956,0.019452433858149397,50000.0,4.504882194106331,750000 +0.12880251575604057,991.78,48.5066139820128,100.67738649711879,5.736007310869006,-410.44884325073366,0.005090077952332755,38.71615002689411,1.2316939807009775,2.014631623496999,0.0006380661624080842,0.02007853209978696,50000.0,4.498189492158401,800000 +0.13278262871906055,1000.0,0.0,104.57425060185217,0.35575791830512266,-408.8335219671676,0.007581578372567769,40.361323673706345,1.2687317558646096,2.0404345441460627,0.0005872772678683231,0.019144103259774177,50000.0,4.499245201930373,850000 +0.1336375717510294,1000.0,0.0,99.74442700754601,0.7227105689624482,-410.76937469848855,-0.0094184920758381,40.89544262109775,1.283189957686668,2.0503697872257294,0.0005903885926768735,0.019221713272497463,50000.0,4.49750979012269,900000 +0.13440783154964436,1000.0,0.0,110.18319968991217,1.0150374268067637,-413.48732143798725,0.0011077867784351045,42.00336886980111,1.291615564835063,2.0953517207908305,0.0006287330631685561,0.01964543524747992,50000.0,4.503623869802689,950000 +0.13443199496030914,974.7,90.51878258129635,107.69768328164704,10.81737878587162,-411.7819143768281,0.0017480202572048298,41.89560525329612,1.3091658001685058,2.079487475335588,0.0006098181039439399,0.019403298532253723,50000.0,4.501886708077336,1000000 +0.13444870539576018,957.49,118.5490189752745,100.08642209053332,15.6741903758439,-408.5434894622854,0.0011664184267819546,42.70844854494166,1.3222564698982118,2.108094249390377,0.0006342621330264607,0.020007831796031342,50000.0,4.498596252058717,1050000 +0.1308416971166419,1000.0,0.0,112.49217682638195,1.0131085819644428,-411.0305123486309,-0.0049848708545417115,43.03835876981761,1.3150432666790182,2.103111256698355,0.0005518067279336038,0.01844543564813362,50000.0,4.506202142236534,1100000 +0.1253712296299633,968.25,121.94100007790652,106.78714281142669,14.716634164240327,-411.98903763366354,0.008279219906590909,41.898097761116205,1.3036615682125063,2.123443379194744,0.0006709368154777039,0.02071771799568391,50000.0,4.501094617372684,1150000 +0.12625644824474844,909.81,177.623517305564,100.83705929327778,21.89005443231251,-416.39039361633826,-0.00906150605494176,42.82724965522726,1.321167073609849,2.1602531506132685,0.0006454307031589445,0.020200936001153913,50000.0,4.49879347178669,1200000 +0.1298655966505417,882.0,193.6835047183936,96.504405806907,23.038783107730442,-414.9911108074954,0.003598426792360838,43.98948135757433,1.3454570447730976,2.1791435157167953,0.0007024188755563054,0.02099638451785811,50000.0,4.5031175200597175,1250000 +0.12675039758816334,985.77,62.63127892674713,106.60940089888389,7.720062556534388,-416.8465107488988,-0.0045046105306597315,43.800264824772285,1.3231322814524153,2.184815022845243,0.0006493630813642105,0.020192771482484623,50000.0,4.502563070985958,1300000 +0.12982989075496934,985.2,54.54612726857884,105.04560674661414,7.662492507450858,-414.6989521160872,0.00976809335846454,45.239256614475046,1.347062550463666,2.2466490283012157,0.000619654737248792,0.01978363538545914,50000.0,4.503549275301144,1350000 +0.12871404233321393,911.31,166.70762999934945,98.31158614356626,19.988284588754198,-410.7892950659184,-0.0026638626544175617,43.83984147794776,1.3302610741937153,2.215137284901142,0.0005595644664848979,0.01850261699254701,50000.0,4.504440798566234,1400000 +0.13137337647885022,850.18,174.26332832813677,86.47287785414463,20.12371409593287,-411.81264301880066,-0.0025911244349182145,45.484576283378196,1.3530162369608807,2.2153707198786474,0.0005390822773008259,0.018136813733480445,50000.0,4.504204836944033,1450000 +0.12477122862666909,619.17,219.01297016386948,62.917094548310196,25.911129885823687,-412.6400688824486,0.0030301326893643113,44.88004643043519,1.320268314484368,2.2145427253985583,0.0006120019615519389,0.019333784690756025,50000.0,4.502483630921334,1500000 +0.12505889306127707,929.58,145.70327244094418,102.23954587438257,17.200850570649113,-410.4485754998775,-0.005767515706717927,46.04562319505726,1.3442104278254392,2.2241695684683727,0.0005667174203788729,0.018643328656446172,50000.0,4.504479635916513,1550000 +0.12403296689555002,999.2,5.894064811316549,113.38273648898671,1.149855936104191,-410.3012562927233,-0.0040477737481519034,44.589596363181954,1.3188873177135068,2.218867811210184,0.0005941964650061391,0.01920651781708013,50000.0,4.506663663465029,1600000 +0.12212470653757218,983.57,64.0330000858932,104.82998930762966,7.357942768225271,-411.69568864990015,-0.00418798907689759,44.78258673746117,1.3199428569686478,2.208446806521428,0.0006186512997047081,0.01979791985467033,50000.0,4.504443630407256,1650000 +0.12019846062079066,1000.0,0.0,113.4646945203922,1.2700828718204114,-412.75559671203206,-0.0003607668973504274,45.518826073379124,1.3462269112241334,2.222246083680389,0.0006108439785531834,0.019554309618872688,50000.0,4.500228130342672,1700000 +0.12199667188629394,1000.0,0.0,113.1153314022738,1.3106233920905241,-415.3822407360811,-0.0011228126082197328,45.911296290207126,1.3592444510674342,2.2555140215301503,0.0005756435003794962,0.01894276451028257,50000.0,4.5009709017572295,1750000 +0.12324391163855797,999.58,4.178947235847805,113.27621376603032,1.216402206090331,-416.72611332763665,0.006520021766498767,45.10641899347246,1.3519848985326195,2.2417980802762427,0.0005449875724124868,0.01849275476987606,50000.0,4.50446073873964,1800000 +0.11975238701418034,998.25,17.41228014936585,114.4927768327486,2.850511097994961,-417.1030730798345,-0.01690239723689848,44.00123513206444,1.3119924789679034,2.2251366396177437,0.0005680886526518676,0.019084638548117365,50000.0,4.50231287820525,1850000 +0.12750994316011616,1000.0,0.0,114.6738679299576,1.8398946604072628,-408.01190548217323,0.00882083923861382,47.854938696079266,1.3856822826480895,2.2910430027687902,0.0005669912407208179,0.018807539437941154,50000.0,4.498664832067047,1900000 +0.1204289554001374,1000.0,0.0,115.38413222844306,1.9191944887181072,-413.5728633459451,0.010814416325986278,46.28469824846318,1.362996996790163,2.280578791339406,0.0005776087737634631,0.018979036832497083,50000.0,4.50209987135071,1950000 +0.11995686561807879,1000.0,0.0,114.70297788462874,0.29787495842160694,-412.3203330322259,-0.007357618008516889,45.50674723411514,1.3631847860086004,2.2537372782766893,0.0005514485579907278,0.01849264204048198,50000.0,4.504233423252556,2000000 +0.12564245096638946,1000.0,0.0,113.65158516583934,0.44033847917016183,-412.0752151947053,0.011591182263288605,50.82887468126341,1.4342247298813,2.367570291205601,0.0005261780950907794,0.018026800942420793,50000.0,4.504330768809274,2050000 +0.12492516812220125,1000.0,0.0,115.34151904186125,0.8988768105932453,-412.8927104669237,-0.003950973252617002,48.488353421097244,1.4222012115967217,2.3134692352699933,0.0005305263753706298,0.018135099258592254,50000.0,4.504730903951095,2100000 +0.11873369111671946,903.96,179.79604667511464,102.25672336828835,22.87739592675943,-416.4588667053237,-0.003589783057533199,47.78406488510116,1.3990796468663371,2.2957481451499273,0.0005140479303575937,0.01774688997739213,50000.0,4.505677171387764,2150000 +0.12090336416661746,1000.0,0.0,114.63059840800496,0.5885733373964084,-418.21674797241445,-0.013102148478142997,46.69858019998561,1.3632353650104914,2.2879595446467302,0.0005490548113881242,0.01850055071382719,50000.0,4.505273444308122,2200000 +0.11460016686722684,988.41,58.94948600284824,112.34551859812652,7.28122598894768,-418.5451563665793,0.003385531250313209,45.884922570094815,1.357412989407789,2.2747827185141825,0.0005417544401206311,0.018253316662644577,50000.0,4.504175860228366,2250000 +0.11501892561331441,1000.0,0.0,114.36230239270799,1.100719958347576,-417.14866249816856,0.002869770096689322,45.341069040718345,1.369820572127105,2.263052331318891,0.0005435508485096274,0.018434348209556462,50000.0,4.50659997234271,2300000 +0.12397203587293777,1000.0,0.0,115.81479215732695,0.700128080798876,-417.25078667785044,0.010204134441800681,48.68879030612974,1.4100204474198546,2.3260218592226503,0.0005905087336755766,0.01915070210661019,50000.0,4.508518598814299,2350000 +0.11695031580641788,993.9,37.235869803188436,113.13334214306936,4.407343849469078,-410.98460469238535,-0.005713218359611911,49.62354444461765,1.4220052683603863,2.35304855933308,0.0005238733217341739,0.018052168895425866,50000.0,4.502566697216818,2400000 +0.11423911713272282,1000.0,0.0,115.93352541754729,0.6759880732755235,-414.8187801477056,0.00025774552602326697,50.59070494045303,1.4226161181271137,2.380960634360278,0.0005429471792444708,0.01840354350927679,50000.0,4.502211796137093,2450000 +0.12141991415813566,976.09,80.03288011811146,109.8593256397996,9.919996241998104,-415.731692028197,0.0009036892588809119,52.397285388126456,1.4525190819537375,2.410051106262217,0.0005097171568603374,0.017656328966537235,50000.0,4.5030662385110976,2500000 +0.11744837074950332,999.8,1.98997487421324,118.98531167428838,0.9978804182583181,-414.12019257873254,0.003301991622410723,50.02146774322562,1.4207610461568587,2.3773903437197688,0.0005500374878885748,0.018614900585831762,50000.0,4.5040293864368435,2550000 +0.10914284145727698,1000.0,0.0,115.23458351213011,0.3833268214970506,-413.33716843139814,-0.013852299617417171,48.24656021350794,1.377287008533502,2.3500983429336784,0.0005193071461653665,0.017767866603245353,50000.0,4.499697076283728,2600000 +0.10888866077884976,1000.0,0.0,112.72519787522508,0.7663532382602328,-414.05345266906363,0.011416474301405214,48.97053322200748,1.4034929836309034,2.34272358039977,0.0005237803337630102,0.01792809543511908,50000.0,4.501452674248914,2650000 +0.11315781208261796,1000.0,0.0,114.11730642904199,0.5678749074962044,-415.8663295269772,0.018995217038914214,49.28950697042481,1.4248350696372871,2.350471117910127,0.0005291518477678532,0.01802055690170987,50000.0,4.502903619458416,2700000 +0.11866438102856414,1000.0,0.0,116.55126656366852,0.6034062574663702,-408.6218483886709,-0.01752794914294034,53.3613232378961,1.495748312735568,2.4371340960025933,0.0005196263817802364,0.017960868657260622,50000.0,4.503818896843119,2750000 +0.11604221203863618,1000.0,0.0,117.36885272050816,0.7717053399359715,-409.4275291113268,0.00020515939962078823,55.45823475183378,1.5294564986634225,2.465781612627553,0.0005122441378944204,0.017807367297982633,50000.0,4.50677882196491,2800000 +0.1154893883092693,982.13,67.61281757181844,112.88604907358841,8.45600144932738,-410.7022026971394,-0.0023532940172032693,55.38871046089171,1.514247176069014,2.4683640323662814,0.0005345127266742406,0.01807984913921398,50000.0,4.506292314400282,2850000 +0.10513722440719586,1000.0,0.0,117.01712186750513,0.335179591544388,-411.83552749634146,-0.02362239911705239,55.55467453645692,1.5311787532711227,2.4864559094011818,0.0005186965715144878,0.01792030179126384,50000.0,4.503825430148538,2900000 +0.10643273345157424,1000.0,0.0,116.97259994592659,0.3500293501482068,-408.07276607909785,0.0004039593886589051,52.20588281557074,1.499928885056981,2.4167746071767695,0.0005155854285175211,0.01777009884239078,50000.0,4.508033999696813,2950000 +0.10804673026651203,1000.0,0.0,117.84458352988807,0.32337175330913015,-398.57825217041153,-0.024916383160687842,53.42685119583152,1.5117137115216381,2.427951221086976,0.000519295063125342,0.017984863128195413,50000.0,4.506466542994119,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-010719/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-010719/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..5a711af0fb6325dcce7a98000f98824d90be5192 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-010719/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 4.4136 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5687 +num rollout transitions: 50000, reward mean: 4.5528 +num rollout transitions: 50000, reward mean: 4.5494 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4755 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5360 +num rollout transitions: 50000, reward mean: 4.5253 +num rollout transitions: 50000, reward mean: 4.5363 +num rollout transitions: 50000, reward mean: 4.5377 +num rollout transitions: 50000, reward mean: 4.5404 +num rollout transitions: 50000, reward mean: 4.5464 +num rollout transitions: 50000, reward mean: 4.5292 +num rollout transitions: 50000, reward mean: 4.5520 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.5282 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5320 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5263 +num rollout transitions: 50000, reward mean: 4.5457 +num rollout transitions: 50000, reward mean: 4.5518 +num rollout transitions: 50000, reward mean: 4.5610 +num rollout transitions: 50000, reward mean: 4.5448 +num rollout transitions: 50000, reward mean: 4.5421 +num rollout transitions: 50000, reward mean: 4.5497 +num rollout transitions: 50000, reward mean: 4.5578 +num rollout transitions: 50000, reward mean: 4.5413 +num rollout transitions: 50000, reward mean: 4.5349 +num rollout transitions: 50000, reward mean: 4.5418 +num rollout transitions: 50000, reward mean: 4.5467 +num rollout transitions: 50000, reward mean: 4.5454 +num rollout transitions: 50000, reward mean: 4.5289 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5273 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5114 +---------------------------------------------------------------------------------- +| alpha | 0.266 | +| eval/episode_length | 24 | +| eval/episode_length_std | 10.1 | +| eval/normalized_episode_reward | 0.0373 | +| eval/normalized_episode_reward_std | 0.0629 | +| loss/actor | -286 | +| loss/alpha | -1.79 | +| loss/critic | 20.8 | +| penalty/lcb_mean | 1 | +| penalty/lcb_std | 1.24 | +| rollout_info/dynamics_penalty_mean | 0.00188 | +| rollout_info/dynamics_penalty_std | 0.0319 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.53 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4812 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4779 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.5279 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.4826 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5220 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 117 | +| eval/episode_length_std | 6.5 | +| eval/normalized_episode_reward | 0.321 | +| eval/normalized_episode_reward_std | 0.0674 | +| loss/actor | -416 | +| loss/alpha | 0.0138 | +| loss/critic | 33.8 | +| penalty/lcb_mean | 1.24 | +| penalty/lcb_std | 1.75 | +| rollout_info/dynamics_penalty_mean | 0.00126 | +| rollout_info/dynamics_penalty_std | 0.0289 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5279 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5334 +num rollout transitions: 50000, reward mean: 4.5245 +num rollout transitions: 50000, reward mean: 4.5297 +num rollout transitions: 50000, reward mean: 4.5347 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.5286 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5387 +num rollout transitions: 50000, reward mean: 4.5285 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5280 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5305 +num rollout transitions: 50000, reward mean: 4.5305 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5323 +num rollout transitions: 50000, reward mean: 4.5256 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5453 +---------------------------------------------------------------------------------- +| alpha | 0.136 | +| eval/episode_length | 231 | +| eval/episode_length_std | 64 | +| eval/normalized_episode_reward | 5.77 | +| eval/normalized_episode_reward_std | 6.26 | +| loss/actor | -443 | +| loss/alpha | -0.00399 | +| loss/critic | 31.6 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.68 | +| rollout_info/dynamics_penalty_mean | 0.00131 | +| rollout_info/dynamics_penalty_std | 0.0288 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.52 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5620 +num rollout transitions: 50000, reward mean: 4.5416 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5284 +num rollout transitions: 50000, reward mean: 4.5401 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5257 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5264 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5241 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5313 +num rollout transitions: 50000, reward mean: 4.5234 +num rollout transitions: 50000, reward mean: 4.5370 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5354 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5297 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5039 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 120 | +| eval/episode_length_std | 8.93 | +| eval/normalized_episode_reward | 5.95 | +| eval/normalized_episode_reward_std | 0.735 | +| loss/actor | -443 | +| loss/alpha | 0.00468 | +| loss/critic | 33.3 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 1.74 | +| rollout_info/dynamics_penalty_mean | 0.000867 | +| rollout_info/dynamics_penalty_std | 0.0233 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.52 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5475 +num rollout transitions: 50000, reward mean: 4.5356 +num rollout transitions: 50000, reward mean: 4.5370 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5383 +num rollout transitions: 50000, reward mean: 4.5277 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5266 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5245 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5226 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5174 +---------------------------------------------------------------------------------- +| alpha | 0.143 | +| eval/episode_length | 115 | +| eval/episode_length_std | 3.79 | +| eval/normalized_episode_reward | 5.24 | +| eval/normalized_episode_reward_std | 0.216 | +| loss/actor | -431 | +| loss/alpha | 0.000562 | +| loss/critic | 32.6 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 1.72 | +| rollout_info/dynamics_penalty_mean | 0.000859 | +| rollout_info/dynamics_penalty_std | 0.0232 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5300 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5321 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4884 +---------------------------------------------------------------------------------- +| alpha | 0.148 | +| eval/episode_length | 142 | +| eval/episode_length_std | 7.3 | +| eval/normalized_episode_reward | 7.91 | +| eval/normalized_episode_reward_std | 0.783 | +| loss/actor | -410 | +| loss/alpha | 0.000975 | +| loss/critic | 36.5 | +| penalty/lcb_mean | 1.27 | +| penalty/lcb_std | 1.86 | +| rollout_info/dynamics_penalty_mean | 0.000817 | +| rollout_info/dynamics_penalty_std | 0.0227 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4793 +num rollout transitions: 50000, reward mean: 4.4752 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5241 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5267 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5331 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5099 +---------------------------------------------------------------------------------- +| alpha | 0.145 | +| eval/episode_length | 865 | +| eval/episode_length_std | 179 | +| eval/normalized_episode_reward | 87.7 | +| eval/normalized_episode_reward_std | 19.2 | +| loss/actor | -402 | +| loss/alpha | -0.00275 | +| loss/critic | 33.8 | +| penalty/lcb_mean | 1.24 | +| penalty/lcb_std | 1.77 | +| rollout_info/dynamics_penalty_mean | 0.000854 | +| rollout_info/dynamics_penalty_std | 0.0231 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4794 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4837 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4801 +num rollout transitions: 50000, reward mean: 4.4894 +---------------------------------------------------------------------------------- +| alpha | 0.148 | +| eval/episode_length | 993 | +| eval/episode_length_std | 51.8 | +| eval/normalized_episode_reward | 99.8 | +| eval/normalized_episode_reward_std | 5.83 | +| loss/actor | -396 | +| loss/alpha | 0.00174 | +| loss/critic | 37.5 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 1.88 | +| rollout_info/dynamics_penalty_mean | 0.000859 | +| rollout_info/dynamics_penalty_std | 0.0234 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5337 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5264 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4988 +---------------------------------------------------------------------------------- +| alpha | 0.149 | +| eval/episode_length | 815 | +| eval/episode_length_std | 308 | +| eval/normalized_episode_reward | 81.7 | +| eval/normalized_episode_reward_std | 33.6 | +| loss/actor | -390 | +| loss/alpha | 0.00634 | +| loss/critic | 40.7 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2 | +| rollout_info/dynamics_penalty_mean | 0.00085 | +| rollout_info/dynamics_penalty_std | 0.0234 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5387 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4776 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.4811 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5320 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4780 +num rollout transitions: 50000, reward mean: 4.4802 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5337 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5193 +---------------------------------------------------------------------------------- +| alpha | 0.158 | +| eval/episode_length | 942 | +| eval/episode_length_std | 107 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 13 | +| loss/actor | -385 | +| loss/alpha | 0.00279 | +| loss/critic | 43.4 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.05 | +| rollout_info/dynamics_penalty_mean | 0.000765 | +| rollout_info/dynamics_penalty_std | 0.0221 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5274 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5244 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4977 +---------------------------------------------------------------------------------- +| alpha | 0.155 | +| eval/episode_length | 966 | +| eval/episode_length_std | 92.5 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 11.5 | +| loss/actor | -385 | +| loss/alpha | 0.00418 | +| loss/critic | 43.8 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.07 | +| rollout_info/dynamics_penalty_mean | 0.0007 | +| rollout_info/dynamics_penalty_std | 0.021 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.4785 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4847 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5015 +---------------------------------------------------------------------------------- +| alpha | 0.149 | +| eval/episode_length | 663 | +| eval/episode_length_std | 149 | +| eval/normalized_episode_reward | 69.2 | +| eval/normalized_episode_reward_std | 17.8 | +| loss/actor | -387 | +| loss/alpha | -0.0175 | +| loss/critic | 45.2 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.12 | +| rollout_info/dynamics_penalty_mean | 0.00085 | +| rollout_info/dynamics_penalty_std | 0.0233 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4831 +num rollout transitions: 50000, reward mean: 4.4709 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4819 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5026 +---------------------------------------------------------------------------------- +| alpha | 0.144 | +| eval/episode_length | 920 | +| eval/episode_length_std | 144 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 17.7 | +| loss/actor | -388 | +| loss/alpha | 0.0041 | +| loss/critic | 43.6 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.09 | +| rollout_info/dynamics_penalty_mean | 0.000762 | +| rollout_info/dynamics_penalty_std | 0.022 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4831 +num rollout transitions: 50000, reward mean: 4.4714 +num rollout transitions: 50000, reward mean: 4.4811 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.4790 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4799 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4797 +num rollout transitions: 50000, reward mean: 4.4933 +---------------------------------------------------------------------------------- +| alpha | 0.143 | +| eval/episode_length | 863 | +| eval/episode_length_std | 194 | +| eval/normalized_episode_reward | 96.1 | +| eval/normalized_episode_reward_std | 23.6 | +| loss/actor | -383 | +| loss/alpha | 0.00246 | +| loss/critic | 45.1 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.13 | +| rollout_info/dynamics_penalty_mean | 0.000688 | +| rollout_info/dynamics_penalty_std | 0.0209 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4749 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4723 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5264 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5222 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4804 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4764 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5110 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 995 | +| eval/episode_length_std | 41 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 4.75 | +| loss/actor | -386 | +| loss/alpha | -0.00096 | +| loss/critic | 44.2 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.15 | +| rollout_info/dynamics_penalty_mean | 0.000712 | +| rollout_info/dynamics_penalty_std | 0.0214 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4812 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4814 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.4794 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.4977 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 977 | +| eval/episode_length_std | 95.7 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 11.4 | +| loss/actor | -386 | +| loss/alpha | -0.00572 | +| loss/critic | 44.8 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.14 | +| rollout_info/dynamics_penalty_mean | 0.000711 | +| rollout_info/dynamics_penalty_std | 0.0214 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5217 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4827 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5205 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5319 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5062 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 947 | +| eval/episode_length_std | 121 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 14.8 | +| loss/actor | -385 | +| loss/alpha | 0.00656 | +| loss/critic | 47 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.2 | +| rollout_info/dynamics_penalty_mean | 0.000685 | +| rollout_info/dynamics_penalty_std | 0.0209 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4778 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5028 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 960 | +| eval/episode_length_std | 101 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 11.9 | +| loss/actor | -382 | +| loss/alpha | -0.0146 | +| loss/critic | 46.9 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.19 | +| rollout_info/dynamics_penalty_mean | 0.000682 | +| rollout_info/dynamics_penalty_std | 0.0207 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4803 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.4819 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5072 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.817 | +| loss/actor | -380 | +| loss/alpha | 0.0167 | +| loss/critic | 48.9 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.26 | +| rollout_info/dynamics_penalty_mean | 0.000679 | +| rollout_info/dynamics_penalty_std | 0.0207 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4833 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5258 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4774 +num rollout transitions: 50000, reward mean: 4.4727 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4786 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4962 +---------------------------------------------------------------------------------- +| alpha | 0.146 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.659 | +| loss/actor | -379 | +| loss/alpha | -0.00687 | +| loss/critic | 51.7 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 0.000626 | +| rollout_info/dynamics_penalty_std | 0.0199 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4732 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4765 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4700 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4816 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4735 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4857 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5026 +---------------------------------------------------------------------------------- +| alpha | 0.143 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 1.5 | +| loss/actor | -379 | +| loss/alpha | 0.00516 | +| loss/critic | 51.1 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.29 | +| rollout_info/dynamics_penalty_mean | 0.000628 | +| rollout_info/dynamics_penalty_std | 0.0198 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4791 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4672 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4828 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4785 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4986 +---------------------------------------------------------------------------------- +| alpha | 0.139 | +| eval/episode_length | 971 | +| eval/episode_length_std | 90.3 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 10.7 | +| loss/actor | -382 | +| loss/alpha | -0.00254 | +| loss/critic | 52.5 | +| penalty/lcb_mean | 1.55 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 0.000639 | +| rollout_info/dynamics_penalty_std | 0.0201 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4803 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4757 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4706 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4814 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4767 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4832 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4857 +num rollout transitions: 50000, reward mean: 4.4826 +num rollout transitions: 50000, reward mean: 4.4853 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 517 | +| eval/episode_length_std | 199 | +| eval/normalized_episode_reward | 52.4 | +| eval/normalized_episode_reward_std | 23.5 | +| loss/actor | -384 | +| loss/alpha | -0.00114 | +| loss/critic | 56.3 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0.000669 | +| rollout_info/dynamics_penalty_std | 0.0205 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4843 +num rollout transitions: 50000, reward mean: 4.4796 +num rollout transitions: 50000, reward mean: 4.4831 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4819 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4675 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4840 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5076 +----------------------------------------------------------------------------------- +| alpha | 0.142 | +| eval/episode_length | 972 | +| eval/episode_length_std | 86.4 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 10.7 | +| loss/actor | -387 | +| loss/alpha | -0.000733 | +| loss/critic | 55.4 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 0.000716 | +| rollout_info/dynamics_penalty_std | 0.0213 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1200000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4823 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.4749 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4832 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5108 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 1.68 | +| loss/actor | -384 | +| loss/alpha | 0.00208 | +| loss/critic | 54 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.37 | +| rollout_info/dynamics_penalty_mean | 0.000679 | +| rollout_info/dynamics_penalty_std | 0.0209 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4753 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4788 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.4775 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4763 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4837 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4723 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4866 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 1.2 | +| loss/actor | -385 | +| loss/alpha | 0.00381 | +| loss/critic | 56.6 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0.000633 | +| rollout_info/dynamics_penalty_std | 0.0201 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4741 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5255 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4824 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4831 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4672 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4798 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4826 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5106 +---------------------------------------------------------------------------------- +| alpha | 0.143 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.71 | +| loss/actor | -384 | +| loss/alpha | -0.00609 | +| loss/critic | 55.9 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0.000686 | +| rollout_info/dynamics_penalty_std | 0.0209 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.4718 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4733 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4828 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4863 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 1.51 | +| loss/actor | -390 | +| loss/alpha | 0.00112 | +| loss/critic | 54.4 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 0.000772 | +| rollout_info/dynamics_penalty_std | 0.0223 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4747 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4766 +num rollout transitions: 50000, reward mean: 4.4663 +num rollout transitions: 50000, reward mean: 4.4702 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5017 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 982 | +| eval/episode_length_std | 104 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 13.4 | +| loss/actor | -387 | +| loss/alpha | 0.00875 | +| loss/critic | 56.5 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.44 | +| rollout_info/dynamics_penalty_mean | 0.000662 | +| rollout_info/dynamics_penalty_std | 0.0203 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4843 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4668 +num rollout transitions: 50000, reward mean: 4.4768 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4701 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4923 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.621 | +| loss/actor | -388 | +| loss/alpha | -0.0083 | +| loss/critic | 54.7 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.39 | +| rollout_info/dynamics_penalty_mean | 0.000682 | +| rollout_info/dynamics_penalty_std | 0.0209 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4801 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4767 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4843 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4832 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.4827 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4740 +num rollout transitions: 50000, reward mean: 4.4833 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5056 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 1 | +| loss/actor | -388 | +| loss/alpha | -0.00314 | +| loss/critic | 58.1 | +| penalty/lcb_mean | 1.63 | +| penalty/lcb_std | 2.47 | +| rollout_info/dynamics_penalty_mean | 0.000641 | +| rollout_info/dynamics_penalty_std | 0.02 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4802 +num rollout transitions: 50000, reward mean: 4.4788 +num rollout transitions: 50000, reward mean: 4.4779 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.4761 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4811 +num rollout transitions: 50000, reward mean: 4.4784 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4730 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4787 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.4946 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.424 | +| loss/actor | -388 | +| loss/alpha | -0.00519 | +| loss/critic | 53.6 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 0.000643 | +| rollout_info/dynamics_penalty_std | 0.0203 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4657 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4738 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.4775 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4833 +num rollout transitions: 50000, reward mean: 4.5255 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5098 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.708 | +| loss/actor | -388 | +| loss/alpha | -0.00494 | +| loss/critic | 54.6 | +| penalty/lcb_mean | 1.58 | +| penalty/lcb_std | 2.4 | +| rollout_info/dynamics_penalty_mean | 0.000604 | +| rollout_info/dynamics_penalty_std | 0.0195 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5221 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4704 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5289 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4961 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 0.455 | +| loss/actor | -391 | +| loss/alpha | 0.0131 | +| loss/critic | 52.9 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 0.000625 | +| rollout_info/dynamics_penalty_std | 0.0198 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4835 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4703 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4807 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4758 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.4961 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.455 | +| loss/actor | -389 | +| loss/alpha | -0.0186 | +| loss/critic | 52.9 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.34 | +| rollout_info/dynamics_penalty_mean | 0.000593 | +| rollout_info/dynamics_penalty_std | 0.0192 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4843 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.4759 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4680 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.5222 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4925 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 0.838 | +| loss/actor | -390 | +| loss/alpha | -0.00383 | +| loss/critic | 49.8 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.31 | +| rollout_info/dynamics_penalty_mean | 0.000667 | +| rollout_info/dynamics_penalty_std | 0.0205 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4798 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4804 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4926 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.481 | +| loss/actor | -392 | +| loss/alpha | 0.0222 | +| loss/critic | 52.6 | +| penalty/lcb_mean | 1.56 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.000646 | +| rollout_info/dynamics_penalty_std | 0.0202 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4697 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4793 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4832 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4826 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4918 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 0.676 | +| loss/actor | -384 | +| loss/alpha | -0.00788 | +| loss/critic | 52.5 | +| penalty/lcb_mean | 1.56 | +| penalty/lcb_std | 2.33 | +| rollout_info/dynamics_penalty_mean | 0.000644 | +| rollout_info/dynamics_penalty_std | 0.0201 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4744 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4799 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5300 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4936 +----------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 997 | +| eval/episode_length_std | 23 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 3.16 | +| loss/actor | -386 | +| loss/alpha | -0.000955 | +| loss/critic | 52.9 | +| penalty/lcb_mean | 1.57 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.000588 | +| rollout_info/dynamics_penalty_std | 0.0193 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1950000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4812 +num rollout transitions: 50000, reward mean: 4.4828 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4702 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4814 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4801 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4798 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4897 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.854 | +| loss/actor | -387 | +| loss/alpha | 0.00107 | +| loss/critic | 58.7 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.44 | +| rollout_info/dynamics_penalty_mean | 0.000602 | +| rollout_info/dynamics_penalty_std | 0.0193 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4760 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4901 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 0.506 | +| loss/actor | -386 | +| loss/alpha | 0.00233 | +| loss/critic | 56.7 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0.00064 | +| rollout_info/dynamics_penalty_std | 0.02 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4777 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4867 +num rollout transitions: 50000, reward mean: 4.4701 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.4847 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4784 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5024 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 988 | +| eval/episode_length_std | 55.3 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 6.7 | +| loss/actor | -386 | +| loss/alpha | -0.0168 | +| loss/critic | 57.1 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0.000657 | +| rollout_info/dynamics_penalty_std | 0.0205 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4825 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4814 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4795 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5009 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.966 | +| loss/actor | -386 | +| loss/alpha | 0.0134 | +| loss/critic | 56.7 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.41 | +| rollout_info/dynamics_penalty_mean | 0.000677 | +| rollout_info/dynamics_penalty_std | 0.0207 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4795 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4783 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4745 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4835 +num rollout transitions: 50000, reward mean: 4.4680 +num rollout transitions: 50000, reward mean: 4.4724 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4820 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4969 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 997 | +| eval/episode_length_std | 26.2 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 3.53 | +| loss/actor | -386 | +| loss/alpha | -0.00965 | +| loss/critic | 56.7 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.44 | +| rollout_info/dynamics_penalty_mean | 0.000587 | +| rollout_info/dynamics_penalty_std | 0.0192 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4773 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.4812 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.4823 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4750 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4790 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4756 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4791 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.4932 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 891 | +| eval/episode_length_std | 185 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 23 | +| loss/actor | -388 | +| loss/alpha | 0.00297 | +| loss/critic | 56.7 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0.000584 | +| rollout_info/dynamics_penalty_std | 0.0193 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4815 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4760 +num rollout transitions: 50000, reward mean: 4.4746 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4804 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.4800 +num rollout transitions: 50000, reward mean: 4.4808 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4734 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4713 +num rollout transitions: 50000, reward mean: 4.4732 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4837 +num rollout transitions: 50000, reward mean: 4.4772 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4777 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.786 | +| loss/actor | -388 | +| loss/alpha | 0.00463 | +| loss/critic | 57.1 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.47 | +| rollout_info/dynamics_penalty_mean | 0.000652 | +| rollout_info/dynamics_penalty_std | 0.0204 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.4825 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.4807 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4776 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4755 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4803 +num rollout transitions: 50000, reward mean: 4.4763 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4826 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 997 | +| eval/episode_length_std | 23.5 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 2.92 | +| loss/actor | -387 | +| loss/alpha | -0.00906 | +| loss/critic | 55.3 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.43 | +| rollout_info/dynamics_penalty_mean | 0.000586 | +| rollout_info/dynamics_penalty_std | 0.0191 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4775 +num rollout transitions: 50000, reward mean: 4.4799 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.4794 +num rollout transitions: 50000, reward mean: 4.4837 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4791 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4782 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4785 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4751 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4935 +---------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 998 | +| eval/episode_length_std | 16.4 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 2.18 | +| loss/actor | -386 | +| loss/alpha | 0.0109 | +| loss/critic | 58.7 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.52 | +| rollout_info/dynamics_penalty_mean | 0.000642 | +| rollout_info/dynamics_penalty_std | 0.0201 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4779 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4771 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4840 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4784 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4835 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5249 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4897 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.288 | +| loss/actor | -385 | +| loss/alpha | -0.00307 | +| loss/critic | 58.3 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.5 | +| rollout_info/dynamics_penalty_mean | 0.000617 | +| rollout_info/dynamics_penalty_std | 0.0196 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4795 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4730 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4857 +num rollout transitions: 50000, reward mean: 4.4856 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4750 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4746 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4824 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4847 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4874 +----------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 1.75 | +| loss/actor | -384 | +| loss/alpha | -0.000846 | +| loss/critic | 56.6 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.47 | +| rollout_info/dynamics_penalty_mean | 0.000623 | +| rollout_info/dynamics_penalty_std | 0.0197 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2500000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4823 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4819 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4712 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.5008 +---------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 4.28 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 1.8 | +| loss/actor | -385 | +| loss/alpha | -0.0142 | +| loss/critic | 56.2 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0.000646 | +| rollout_info/dynamics_penalty_std | 0.0201 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.49 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4762 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4804 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.4825 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4672 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.4857 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5050 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 990 | +| eval/episode_length_std | 67.6 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 8.62 | +| loss/actor | -383 | +| loss/alpha | 0.00867 | +| loss/critic | 56.5 | +| penalty/lcb_mean | 1.67 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0.00058 | +| rollout_info/dynamics_penalty_std | 0.0191 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4801 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4831 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4773 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5179 +----------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 968 | +| eval/episode_length_std | 107 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 13.2 | +| loss/actor | -381 | +| loss/alpha | -0.000693 | +| loss/critic | 55.9 | +| penalty/lcb_mean | 1.65 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0.00056 | +| rollout_info/dynamics_penalty_std | 0.0185 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2650000 | +----------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5036 +---------------------------------------------------------------------------------- +| alpha | 0.118 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.561 | +| loss/actor | -379 | +| loss/alpha | -0.00392 | +| loss/critic | 55.5 | +| penalty/lcb_mean | 1.66 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0.000623 | +| rollout_info/dynamics_penalty_std | 0.0198 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4718 +num rollout transitions: 50000, reward mean: 4.4800 +num rollout transitions: 50000, reward mean: 4.4783 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5110 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 969 | +| eval/episode_length_std | 84 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 9.96 | +| loss/actor | -384 | +| loss/alpha | -0.00287 | +| loss/critic | 57.8 | +| penalty/lcb_mean | 1.68 | +| penalty/lcb_std | 2.52 | +| rollout_info/dynamics_penalty_mean | 0.000564 | +| rollout_info/dynamics_penalty_std | 0.0188 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4764 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4810 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.4963 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 874 | +| eval/episode_length_std | 162 | +| eval/normalized_episode_reward | 99.2 | +| eval/normalized_episode_reward_std | 20.6 | +| loss/actor | -378 | +| loss/alpha | 0.0117 | +| loss/critic | 60 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.55 | +| rollout_info/dynamics_penalty_mean | 0.000566 | +| rollout_info/dynamics_penalty_std | 0.0189 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4789 +num rollout transitions: 50000, reward mean: 4.4823 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4742 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.4823 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5004 +---------------------------------------------------------------------------------- +| alpha | 0.116 | +| eval/episode_length | 976 | +| eval/episode_length_std | 109 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 13.1 | +| loss/actor | -377 | +| loss/alpha | -0.0048 | +| loss/critic | 58.6 | +| penalty/lcb_mean | 1.69 | +| penalty/lcb_std | 2.52 | +| rollout_info/dynamics_penalty_mean | 0.00053 | +| rollout_info/dynamics_penalty_std | 0.018 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4821 +num rollout transitions: 50000, reward mean: 4.4817 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4853 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4901 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 970 | +| eval/episode_length_std | 106 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 12.9 | +| loss/actor | -379 | +| loss/alpha | 0.00489 | +| loss/critic | 60.7 | +| penalty/lcb_mean | 1.74 | +| penalty/lcb_std | 2.58 | +| rollout_info/dynamics_penalty_mean | 0.00054 | +| rollout_info/dynamics_penalty_std | 0.0182 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4837 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4930 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 988 | +| eval/episode_length_std | 50.9 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 6.12 | +| loss/actor | -377 | +| loss/alpha | -0.0115 | +| loss/critic | 60.2 | +| penalty/lcb_mean | 1.73 | +| penalty/lcb_std | 2.55 | +| rollout_info/dynamics_penalty_mean | 0.000511 | +| rollout_info/dynamics_penalty_std | 0.0179 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4949 +---------------------------------------------------------------------------------- +| alpha | 0.119 | +| eval/episode_length | 984 | +| eval/episode_length_std | 72.1 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 9.01 | +| loss/actor | -364 | +| loss/alpha | -0.00603 | +| loss/critic | 56.8 | +| penalty/lcb_mean | 1.7 | +| penalty/lcb_std | 2.45 | +| rollout_info/dynamics_penalty_mean | 0.00053 | +| rollout_info/dynamics_penalty_std | 0.0181 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 144557.65s diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-010719/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-010719/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-010719/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-010719/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..88509c05a95ffd17c24cc2c77307105dab09a711 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-010719/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_vae.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 42, "device": "cuda:4", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_42×tamp_26-0112-030600/model", "reward_penalty_coef": 0.7, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/vae", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "vae", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "vae"} diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-010719/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-010719/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..6c17579a382a795d028bd9ec352eff2f8cd8f46b --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/vae/seed_42×tamp_26-0317-010719/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.2661223961170012,23.99,10.09108021967916,0.037262726422868674,0.06286372469544797,-286.43333148526057,-1.7921051862540314,20.77765251832906,1.0034377339765383,1.2400888511193862,0.0018837304681201459,0.03188556839439869,50000.0,4.526131687510331,50000 +0.14001091217517883,116.62,6.5035067463638425,0.321436940043796,0.06740108667715819,-416.4253364160137,0.013792254142742646,33.780806424503325,1.2415174411725953,1.7454243340182432,0.0012573400165890576,0.028865462247886407,50000.0,4.501822010623963,100000 +0.13625091014176582,231.23,63.977160768511766,5.770546749797301,6.25790498077084,-443.23445256408957,-0.003994682304859067,31.609245130157362,1.2658378216183244,1.6810844874536957,0.0013132741201614857,0.028846836179542434,50000.0,4.518149850660899,150000 +0.14542730168074486,119.88,8.92555880603562,5.9542595627132116,0.7348020262337331,-443.3937189508026,0.004678129779584803,33.34421920526452,1.2993764491117048,1.735473324371563,0.0008670515636074648,0.023272935762937767,50000.0,4.515912488405828,200000 +0.1434603705477704,115.45,3.785168424258028,5.241582754339632,0.21596058877798216,-430.7366050524892,0.0005618166324868989,32.59630173465732,1.259551158877607,1.7176744942784237,0.0008592435724121162,0.023175303522486482,50000.0,4.511115838839463,250000 +0.14785173324167555,141.53,7.3026775911305295,7.907573223050181,0.7831218082190996,-410.2646131103551,0.0009746494709701015,36.52115928348571,1.270810832804452,1.8629031439268648,0.0008171919494124946,0.02271652655101745,50000.0,4.5066153959159285,300000 +0.14497240123659416,865.39,179.4874867504696,87.71900537098134,19.217322666100753,-402.16765645202435,-0.002750632561519685,33.82064291849109,1.2396026017391484,1.7691429008805815,0.0008538262105113016,0.023060464065110305,50000.0,4.505594042045798,350000 +0.14784867197364532,992.65,51.76782301777814,99.7854950002297,5.830624184016211,-396.3256689904798,0.0017430433387309885,37.54347713703145,1.2887085317170803,1.8774658053493682,0.0008589928095674377,0.02339912203327205,50000.0,4.501984416097088,400000 +0.14882810612290862,814.6,307.8128327409369,81.67633701243547,33.62872753703598,-389.6675086651601,0.006343860760554683,40.742267645473824,1.3465789948415945,2.004360458519459,0.0008500167114348056,0.023442386183442587,50000.0,4.504943353413747,450000 +0.15823189811229593,942.16,107.18290162148067,105.15793396975164,12.975234085302755,-385.05074342285025,0.0027881348882988744,43.4317275866522,1.3897347009551846,2.052503267141551,0.0007646650172058245,0.022118244084383856,50000.0,4.505095841173583,500000 +0.155239089766142,965.56,92.54645536161827,105.89242380712282,11.464200987734154,-384.849922757566,0.004182887839525955,43.76534270832056,1.3855544071996149,2.0724085502100076,0.0007003231382976462,0.0210289826525131,50000.0,4.502615781152553,550000 +0.1494328083837052,663.42,149.33192424930445,69.20469679265176,17.826755582883578,-386.96929987487385,-0.01754017713759102,45.207590853633775,1.4072753857159672,2.1227911293184607,0.000849737093636312,0.02332938570860154,50000.0,4.502449563288138,600000 +0.14400600793510523,920.09,143.8290022909149,103.94882697662997,17.729807412116124,-387.86738585021584,0.0040997996795736754,43.640613060798756,1.4041804668855773,2.0924971305179456,0.0007619842281794321,0.022005523603587023,50000.0,4.5010115679329274,650000 +0.14285670497536895,862.61,193.59885820944297,96.1213375565655,23.57814401559425,-383.1349914758273,0.002461707372181,45.102154288406695,1.4220348637735944,2.131408919262885,0.0006881088665550457,0.020860455982578706,50000.0,4.502691811116612,700000 +0.13967410486280793,994.63,40.98552305387842,108.9330559872842,4.750303906629883,-386.17099554748216,-0.0009604728571696292,44.23440307006879,1.4106686758196327,2.1450949675202224,0.0007119545092876473,0.021355732379170298,50000.0,4.499076752092551,750000 +0.1384397430256017,977.38,95.65728200194694,107.42167699330534,11.407300365440257,-385.5974500439491,-0.0057160208114981435,44.81386852895801,1.4177764355778484,2.1362314357113936,0.0007112162975462588,0.021371906753548168,50000.0,4.500622131584533,800000 +0.14021785263955444,946.79,120.5607145798332,105.01774418857525,14.808876440136745,-385.0717822350977,0.006559650714881594,47.01822905439262,1.4552488459158035,2.1986649759174135,0.0006845235877113736,0.020897664255258276,50000.0,4.506043698308469,850000 +0.1412666541725363,959.96,100.6481912405782,104.06222168952195,11.92435471332996,-382.12655920836806,-0.014636373293697848,46.93920472787841,1.480568014117477,2.18500796747683,0.0006819271970198434,0.020713911324018973,50000.0,4.501185925242318,900000 +0.13909932191878702,1000.0,0.0,111.2514370200151,0.8168523675451196,-379.6286081469744,0.016666683060042587,48.89940085292778,1.4958892639946837,2.2635041771792963,0.0006791217011670366,0.020707948371383282,50000.0,4.499390049832942,950000 +0.14583124662846306,1000.0,0.0,111.69396795311901,0.6592716552384337,-379.416537569585,-0.006868160633035003,51.73731650854053,1.5399109509277413,2.32909134830953,0.0006263344345324967,0.019885029709484886,50000.0,4.500643169294103,1000000 +0.14320050065994155,1000.0,0.0,105.79680212261273,1.4956966772866365,-379.28660042418767,0.005161089969109798,51.07357058044374,1.549161067628859,2.2906704138934706,0.0006275866045307368,0.01984506750234316,50000.0,4.49509572823354,1050000 +0.139469554614722,970.83,90.32165354996552,105.42089540388788,10.72644872868255,-381.75793394958407,-0.0025359791951627985,52.472644633216944,1.5546988616943251,2.3250259522044523,0.0006389831193224877,0.020119324551519466,50000.0,4.498009739492624,1100000 +0.14087124317854896,516.82,199.19103293070197,52.39894882928233,23.52136308358245,-383.88796033691204,-0.0011394361552968294,56.274743300590586,1.60912891324281,2.4061854155373763,0.0006692173440559421,0.020477657962533452,50000.0,4.495638646650793,1150000 +0.14214496553152814,972.46,86.41520930947283,110.52634700193352,10.666025608813703,-387.4423700268606,-0.000733320169299847,55.39494485767389,1.619314830353261,2.4010635587406197,0.0007160232289293372,0.021294684026681027,50000.0,4.496467946848448,1200000 +0.14000867259621588,1000.0,0.0,108.16823831179885,1.684789552644831,-383.86709830688454,0.0020773690001293344,53.95914197507886,1.5966426801276306,2.366399397287398,0.0006792661948527863,0.020947771666793926,50000.0,4.495774466489488,1250000 +0.14005526033550264,1000.0,0.0,115.9493134342456,1.1977517012321008,-385.3625397698982,0.0038134226542339217,56.624054241714276,1.6272082264614083,2.461078269963269,0.0006328350002190854,0.02006242126918177,50000.0,4.497149410241732,1300000 +0.14253390765577514,1000.0,0.0,112.2985525924213,0.7100577800897323,-384.183727467042,-0.006087611169628671,55.861659174938225,1.6047766146445528,2.415828496725555,0.0006860141154530575,0.02093943031857046,50000.0,4.496228350951082,1350000 +0.13441091991469095,1000.0,0.0,107.01277889629792,1.5120759938664046,-390.0779501879943,0.0011246485051884954,54.41634376934032,1.5881778070330725,2.401928425347841,0.0007719594122638665,0.022329711027737217,50000.0,4.497136435321072,1400000 +0.14056904577732268,981.64,104.40052873429332,112.35867711382717,13.37686664626531,-387.2852971984863,0.00874997883137311,56.48138936218294,1.63630757599352,2.43808771298166,0.0006616787818943849,0.02033280660012871,50000.0,4.494974017558594,1450000 +0.13435602235540883,1000.0,0.0,110.9065017875378,0.6212760085507292,-388.12538909180137,-0.008302561906799328,54.660180076064535,1.584676736216532,2.392964341258991,0.0006817479839118899,0.020858965677599917,50000.0,4.4971607666530895,1500000 +0.1409134576338516,1000.0,0.0,108.1222320512105,1.0013521186901098,-387.9416569036887,-0.0031371670627594623,58.057935023327374,1.6257322572445698,2.470500095944376,0.0006410774710883849,0.020015188491155852,50000.0,4.49564595389024,1550000 +0.13337484966814772,1000.0,0.0,109.09625067500495,0.4244986912728192,-387.5753665057426,-0.005186119405701672,53.614180262775676,1.5693893964243217,2.3806785304236158,0.0006433011811733512,0.02027526457747529,50000.0,4.494221659691053,1600000 +0.13307562694594408,1000.0,0.0,109.66615572129027,0.7084988127769379,-387.9126493127422,-0.004940486404225079,54.557802511329314,1.578932375283256,2.3960741339623812,0.0006038285027862993,0.019450238749811374,50000.0,4.496723474159795,1650000 +0.13445021147593472,1000.0,0.0,106.66783961777759,0.45518878292822634,-391.4599824218808,0.013129431134462452,52.87875416471487,1.569197417039879,2.326763980352892,0.0006246639172006168,0.019809257115399735,50000.0,4.502847094786066,1700000 +0.1328742896884705,1000.0,0.0,108.53285574150952,0.45462252146511345,-388.8725151977533,-0.018574310089387043,52.89871714403146,1.5687979081869161,2.3375294452976907,0.0005926212464649804,0.019178243778323684,50000.0,4.496844720763911,1750000 +0.12458335871383744,1000.0,0.0,111.31185943578524,0.8379586837686246,-390.1080836065695,-0.003833710114732432,49.84775476587311,1.5314379365968729,2.3056650537526098,0.0006669979170218926,0.020546665355571248,50000.0,4.497293715799337,1800000 +0.12614920190244902,1000.0,0.0,111.51056234154268,0.48058110982393343,-392.4971594757047,0.022185643366538717,52.55657851478536,1.5606560194420689,2.351289681203364,0.000645837512603175,0.020159422157425808,50000.0,4.496767171910144,1850000 +0.12969740864574958,1000.0,0.0,114.78132904397698,0.6759793948579188,-383.68351471863514,-0.007878958658948785,52.5254579637518,1.5586680053663196,2.328990092499242,0.0006437615012148363,0.02011971858461711,50000.0,4.494766186979477,1900000 +0.12700563239186904,997.15,22.998423859038688,108.49418581488672,3.162413276090756,-386.28620171752556,-0.0009545887136458763,52.945533838786986,1.571644954450158,2.354271510741648,0.0005881837280157834,0.019251225711579084,50000.0,4.498445349823681,1950000 +0.1350568594431895,1000.0,0.0,112.49202568152353,0.8540139839292511,-386.74509397521945,0.001069765891581766,58.651263711547934,1.6727067256832036,2.4431714264249815,0.0006016379414875946,0.019331516426241323,50000.0,4.49189817826727,2000000 +0.13263819403529362,1000.0,0.0,108.23049859976163,0.5061669996941603,-386.2882769134528,0.0023347416146099437,56.745375631447004,1.6477509514737403,2.416311650105718,0.0006395203439600971,0.019970771796705977,50000.0,4.497246363524573,2050000 +0.13069209754943878,987.59,55.32397220012316,111.86044589864832,6.701380013098386,-386.32058794188976,-0.01676701772380627,57.117799712657956,1.6605820747685558,2.4082965024721315,0.0006567095612204023,0.0204585150878736,50000.0,4.497268007450333,2100000 +0.12851488250091547,1000.0,0.0,113.05750864498636,0.9664574941002493,-385.52456230163625,0.013405771125257152,56.737060309543715,1.6547826648235207,2.411704305996919,0.0006774904319184717,0.0206591186022807,50000.0,4.4970116529116275,2150000 +0.12804597572982204,997.13,26.16205458292602,113.06142700673259,3.533588664101094,-386.1421212969973,-0.009653810757473095,56.72940497537575,1.6572378275513815,2.4364102100062004,0.0005869740448025732,0.019177110733901177,50000.0,4.494596435044718,2200000 +0.12719082541063365,891.19,185.0918526029712,100.61251372888267,23.015696658074337,-388.2005562829628,0.002972151596471766,56.683943780765105,1.64046212211847,2.4541481188559717,0.0005843341955536516,0.019257850839283167,50000.0,4.493837435752513,2250000 +0.12871437010139303,1000.0,0.0,110.17965032148001,0.7859928155368533,-388.35007429869876,0.004629436638727882,57.132669478722,1.6758624694657405,2.472717952771193,0.000651980341977117,0.020401382073245714,50000.0,4.489998697435853,2300000 +0.1251679598252469,996.73,23.528644244834855,109.98852607110004,2.92327777635268,-386.97200913330227,-0.009057841635905164,55.31056336397221,1.6396188242697562,2.4334502630448336,0.0005856778137990275,0.019050192225891688,50000.0,4.492450206633562,2350000 +0.12243644531965191,998.35,16.41729271225923,110.60161195152895,2.1801886881635735,-385.9414688922118,0.010852390843629812,58.72612893409665,1.6616659264445461,2.5181006290698718,0.0006421053542273937,0.020098561723880165,50000.0,4.492796645704305,2400000 +0.12437251887202366,1000.0,0.0,108.71330854779364,0.2882549475016921,-384.59367593688984,-0.0030681784319876635,58.33052220661158,1.7020439484691618,2.5022474641752495,0.0006168556479935999,0.019597178206768692,50000.0,4.494643004905226,2450000 +0.12099355409040985,1000.0,0.0,105.35003447253553,1.7512724763085623,-384.2482136096232,-0.0008462581983580221,56.630986363201075,1.659791213755592,2.4707793775546705,0.000622783514171187,0.01970546089214992,50000.0,4.492916840508318,2500000 +0.12155875778600737,999.57,4.278445979558466,111.79154762820022,1.8034757978635345,-384.63421073303664,-0.014214370256140899,56.18161911710734,1.6596334735894274,2.4493126392078395,0.0006462241770186464,0.020145087858214426,50000.0,4.494472862675382,2550000 +0.11985941510558054,989.67,67.5533944372894,114.02414264630502,8.62259576687611,-383.4036382305876,0.008670037622712592,56.48681587934479,1.6742059989905271,2.44736075321315,0.000579775393644757,0.01912404881523046,50000.0,4.495989528726698,2600000 +0.11928953437998778,968.39,107.04960485681394,109.534225683449,13.202954284063734,-380.66985137084976,-0.000692744279950722,55.92260020921594,1.6476054967379528,2.4538178898906557,0.0005603724060106557,0.018544424200673,50000.0,4.497465764704033,2650000 +0.1179306712217619,1000.0,0.0,112.47575142003889,0.5612842132133977,-379.14139040100156,-0.003916586405075893,55.51135005397848,1.6551811442637319,2.4236676807522888,0.0006231420965932542,0.01975085725014042,50000.0,4.497841982012618,2700000 +0.11670158826246907,968.94,84.00652593697707,106.90923163485371,9.957835370559415,-383.76738372741994,-0.0028653726437313353,57.79835425937693,1.677691380517469,2.516895851471403,0.0005639209192335644,0.018814658829238023,50000.0,4.498037002896121,2750000 +0.12260939910173489,873.83,161.8745844782312,99.22203456770553,20.564429169412026,-378.1908567639191,0.011676405890807404,60.0402064215082,1.7024092489766924,2.5499396436190893,0.0005664194495638291,0.018922427807162957,50000.0,4.497442856207262,2800000 +0.1158238684144627,975.84,109.22607014811068,108.94584326670149,13.118345438375604,-376.86388395324644,-0.0047965829020741495,58.58402171464889,1.6934830800151914,2.5198784754013968,0.0005295941082375472,0.01804077498208474,50000.0,4.498723661365574,2850000 +0.11658778482779783,969.61,106.34292595184692,109.61580171238417,12.93377789747177,-378.74701463134653,0.004893377173934181,60.744306807174304,1.7407614728450755,2.5780832840156624,0.0005395430634442731,0.018216010422344428,50000.0,4.496191433365945,2900000 +0.1170662243522687,988.2,50.90834902056832,112.16637471329318,6.124764417265815,-376.5837653259296,-0.011474218691624807,60.165404864692825,1.734885503787992,2.5528102955461214,0.0005114210442458221,0.01791316114220176,50000.0,4.497223798772187,2950000 +0.11860329829082014,983.95,72.09526683493166,111.96869247235159,9.014001955910791,-364.15445230346836,-0.006034399651065402,56.84903173171972,1.6952737038278705,2.453510211684681,0.0005297916634600661,0.0180840759942769,50000.0,4.498485742716594,3000000 diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-050351/record/consoleout_backup.txt b/walker2d_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-050351/record/consoleout_backup.txt new file mode 100644 index 0000000000000000000000000000000000000000..d46066987897cba669de7584e046dce6ab8fc981 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-050351/record/consoleout_backup.txt @@ -0,0 +1,4021 @@ +num rollout transitions: 50000, reward mean: 4.3911 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5527 +num rollout transitions: 50000, reward mean: 4.5561 +num rollout transitions: 50000, reward mean: 4.5439 +num rollout transitions: 50000, reward mean: 4.5328 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.5286 +num rollout transitions: 50000, reward mean: 4.5284 +num rollout transitions: 50000, reward mean: 4.5323 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5355 +num rollout transitions: 50000, reward mean: 4.5327 +num rollout transitions: 50000, reward mean: 4.5450 +num rollout transitions: 50000, reward mean: 4.5376 +num rollout transitions: 50000, reward mean: 4.5641 +num rollout transitions: 50000, reward mean: 4.5361 +num rollout transitions: 50000, reward mean: 4.5366 +num rollout transitions: 50000, reward mean: 4.5290 +num rollout transitions: 50000, reward mean: 4.5359 +num rollout transitions: 50000, reward mean: 4.5504 +num rollout transitions: 50000, reward mean: 4.5408 +num rollout transitions: 50000, reward mean: 4.5400 +num rollout transitions: 50000, reward mean: 4.5317 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5263 +num rollout transitions: 50000, reward mean: 4.5278 +num rollout transitions: 50000, reward mean: 4.5294 +num rollout transitions: 50000, reward mean: 4.5276 +num rollout transitions: 50000, reward mean: 4.5513 +num rollout transitions: 50000, reward mean: 4.5269 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5457 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5311 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5314 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5366 +num rollout transitions: 50000, reward mean: 4.5253 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5186 +---------------------------------------------------------------------------------- +| alpha | 0.252 | +| eval/episode_length | 387 | +| eval/episode_length_std | 260 | +| eval/normalized_episode_reward | 10.3 | +| eval/normalized_episode_reward_std | 7.7 | +| loss/actor | -288 | +| loss/alpha | -1.77 | +| loss/critic | 18 | +| penalty/lcb_mean | 0.914 | +| penalty/lcb_std | 1.18 | +| rollout_info/dynamics_penalty_mean | 0.00196 | +| rollout_info/dynamics_penalty_std | 0.0305 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.53 | +| timestep | 50000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5268 +num rollout transitions: 50000, reward mean: 4.5208 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4875 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5255 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4745 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4835 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4874 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4775 +num rollout transitions: 50000, reward mean: 4.4825 +num rollout transitions: 50000, reward mean: 4.4759 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.4773 +num rollout transitions: 50000, reward mean: 4.4827 +num rollout transitions: 50000, reward mean: 4.5134 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 189 | +| eval/episode_length_std | 61.1 | +| eval/normalized_episode_reward | 8.82 | +| eval/normalized_episode_reward_std | 6.12 | +| loss/actor | -438 | +| loss/alpha | 0.00887 | +| loss/critic | 28.4 | +| penalty/lcb_mean | 1.13 | +| penalty/lcb_std | 1.66 | +| rollout_info/dynamics_penalty_mean | 0.00117 | +| rollout_info/dynamics_penalty_std | 0.028 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4811 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5258 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5352 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5255 +num rollout transitions: 50000, reward mean: 4.5297 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5294 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5290 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5208 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 295 | +| eval/episode_length_std | 162 | +| eval/normalized_episode_reward | 25.9 | +| eval/normalized_episode_reward_std | 18 | +| loss/actor | -454 | +| loss/alpha | -0.0146 | +| loss/critic | 24 | +| penalty/lcb_mean | 1.05 | +| penalty/lcb_std | 1.57 | +| rollout_info/dynamics_penalty_mean | 0.0011 | +| rollout_info/dynamics_penalty_std | 0.0271 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5391 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5234 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.5240 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5291 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5414 +num rollout transitions: 50000, reward mean: 4.5431 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5393 +num rollout transitions: 50000, reward mean: 4.5311 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5279 +num rollout transitions: 50000, reward mean: 4.5301 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5369 +num rollout transitions: 50000, reward mean: 4.5412 +num rollout transitions: 50000, reward mean: 4.5513 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5291 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5249 +num rollout transitions: 50000, reward mean: 4.5290 +num rollout transitions: 50000, reward mean: 4.5405 +---------------------------------------------------------------------------------- +| alpha | 0.124 | +| eval/episode_length | 396 | +| eval/episode_length_std | 200 | +| eval/normalized_episode_reward | 30.7 | +| eval/normalized_episode_reward_std | 21.4 | +| loss/actor | -450 | +| loss/alpha | 0.017 | +| loss/critic | 25.5 | +| penalty/lcb_mean | 1.07 | +| penalty/lcb_std | 1.58 | +| rollout_info/dynamics_penalty_mean | 0.000922 | +| rollout_info/dynamics_penalty_std | 0.0248 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.52 | +| timestep | 200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.5284 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5252 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.5338 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5203 +num rollout transitions: 50000, reward mean: 4.5281 +num rollout transitions: 50000, reward mean: 4.5253 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5263 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5270 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5274 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5217 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5288 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5346 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.5300 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5294 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5280 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.5270 +---------------------------------------------------------------------------------- +| alpha | 0.137 | +| eval/episode_length | 763 | +| eval/episode_length_std | 275 | +| eval/normalized_episode_reward | 67.3 | +| eval/normalized_episode_reward_std | 27.7 | +| loss/actor | -436 | +| loss/alpha | 0.0143 | +| loss/critic | 30.7 | +| penalty/lcb_mean | 1.14 | +| penalty/lcb_std | 1.72 | +| rollout_info/dynamics_penalty_mean | 0.000714 | +| rollout_info/dynamics_penalty_std | 0.0214 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.52 | +| timestep | 250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5222 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5391 +num rollout transitions: 50000, reward mean: 4.5271 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5244 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5253 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5249 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5217 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5299 +num rollout transitions: 50000, reward mean: 4.5315 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5287 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5284 +num rollout transitions: 50000, reward mean: 4.5252 +num rollout transitions: 50000, reward mean: 4.5261 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5135 +---------------------------------------------------------------------------------- +| alpha | 0.156 | +| eval/episode_length | 193 | +| eval/episode_length_std | 177 | +| eval/normalized_episode_reward | 11.6 | +| eval/normalized_episode_reward_std | 17 | +| loss/actor | -427 | +| loss/alpha | -0.00095 | +| loss/critic | 38.6 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 1.92 | +| rollout_info/dynamics_penalty_mean | 0.000698 | +| rollout_info/dynamics_penalty_std | 0.021 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5354 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5299 +num rollout transitions: 50000, reward mean: 4.5281 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5219 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5219 +num rollout transitions: 50000, reward mean: 4.5264 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5206 +---------------------------------------------------------------------------------- +| alpha | 0.155 | +| eval/episode_length | 903 | +| eval/episode_length_std | 265 | +| eval/normalized_episode_reward | 87.6 | +| eval/normalized_episode_reward_std | 27.2 | +| loss/actor | -414 | +| loss/alpha | 0.00371 | +| loss/critic | 36.5 | +| penalty/lcb_mean | 1.26 | +| penalty/lcb_std | 1.83 | +| rollout_info/dynamics_penalty_mean | 0.000708 | +| rollout_info/dynamics_penalty_std | 0.0214 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5304 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5256 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5314 +num rollout transitions: 50000, reward mean: 4.5290 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5344 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5297 +---------------------------------------------------------------------------------- +| alpha | 0.166 | +| eval/episode_length | 845 | +| eval/episode_length_std | 273 | +| eval/normalized_episode_reward | 82.5 | +| eval/normalized_episode_reward_std | 28.8 | +| loss/actor | -400 | +| loss/alpha | 0.00985 | +| loss/critic | 41.3 | +| penalty/lcb_mean | 1.32 | +| penalty/lcb_std | 1.94 | +| rollout_info/dynamics_penalty_mean | 0.000673 | +| rollout_info/dynamics_penalty_std | 0.0204 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5205 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5285 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5213 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5283 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5263 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5105 +---------------------------------------------------------------------------------- +| alpha | 0.17 | +| eval/episode_length | 698 | +| eval/episode_length_std | 336 | +| eval/normalized_episode_reward | 64.1 | +| eval/normalized_episode_reward_std | 33.7 | +| loss/actor | -398 | +| loss/alpha | 0.00376 | +| loss/critic | 46.8 | +| penalty/lcb_mean | 1.38 | +| penalty/lcb_std | 2.09 | +| rollout_info/dynamics_penalty_mean | 0.000656 | +| rollout_info/dynamics_penalty_std | 0.0201 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5240 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5291 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5227 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5124 +---------------------------------------------------------------------------------- +| alpha | 0.171 | +| eval/episode_length | 912 | +| eval/episode_length_std | 142 | +| eval/normalized_episode_reward | 93.6 | +| eval/normalized_episode_reward_std | 15.5 | +| loss/actor | -396 | +| loss/alpha | -0.0144 | +| loss/critic | 47.5 | +| penalty/lcb_mean | 1.4 | +| penalty/lcb_std | 2.14 | +| rollout_info/dynamics_penalty_mean | 0.000701 | +| rollout_info/dynamics_penalty_std | 0.0209 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5248 +num rollout transitions: 50000, reward mean: 4.5244 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5326 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5257 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5365 +num rollout transitions: 50000, reward mean: 4.5241 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5308 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.5252 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5010 +---------------------------------------------------------------------------------- +| alpha | 0.163 | +| eval/episode_length | 848 | +| eval/episode_length_std | 251 | +| eval/normalized_episode_reward | 80.6 | +| eval/normalized_episode_reward_std | 25.7 | +| loss/actor | -394 | +| loss/alpha | -0.00264 | +| loss/critic | 45.8 | +| penalty/lcb_mean | 1.37 | +| penalty/lcb_std | 2.12 | +| rollout_info/dynamics_penalty_mean | 0.000713 | +| rollout_info/dynamics_penalty_std | 0.0211 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4888 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5291 +num rollout transitions: 50000, reward mean: 4.5293 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5283 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4826 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5283 +num rollout transitions: 50000, reward mean: 4.5222 +num rollout transitions: 50000, reward mean: 4.5251 +num rollout transitions: 50000, reward mean: 4.5005 +---------------------------------------------------------------------------------- +| alpha | 0.154 | +| eval/episode_length | 850 | +| eval/episode_length_std | 170 | +| eval/normalized_episode_reward | 86.3 | +| eval/normalized_episode_reward_std | 19.2 | +| loss/actor | -397 | +| loss/alpha | -0.00457 | +| loss/critic | 45.4 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.15 | +| rollout_info/dynamics_penalty_mean | 0.000758 | +| rollout_info/dynamics_penalty_std | 0.0218 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5334 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5335 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5258 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5253 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5226 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5297 +num rollout transitions: 50000, reward mean: 4.4979 +---------------------------------------------------------------------------------- +| alpha | 0.149 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 1.76 | +| loss/actor | -400 | +| loss/alpha | -0.00187 | +| loss/critic | 44.9 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 2.17 | +| rollout_info/dynamics_penalty_mean | 0.000662 | +| rollout_info/dynamics_penalty_std | 0.0202 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5261 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5408 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.4857 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5412 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5245 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4809 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5287 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5457 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5000 +---------------------------------------------------------------------------------- +| alpha | 0.148 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 105 | +| eval/normalized_episode_reward_std | 1.54 | +| loss/actor | -401 | +| loss/alpha | -0.00478 | +| loss/critic | 45.4 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 2.19 | +| rollout_info/dynamics_penalty_mean | 0.000694 | +| rollout_info/dynamics_penalty_std | 0.0208 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5214 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5266 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5207 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5230 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5240 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5202 +num rollout transitions: 50000, reward mean: 4.5322 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5172 +---------------------------------------------------------------------------------- +| alpha | 0.141 | +| eval/episode_length | 919 | +| eval/episode_length_std | 227 | +| eval/normalized_episode_reward | 99.4 | +| eval/normalized_episode_reward_std | 26.5 | +| loss/actor | -403 | +| loss/alpha | 0.00615 | +| loss/critic | 44.6 | +| penalty/lcb_mean | 1.29 | +| penalty/lcb_std | 2.19 | +| rollout_info/dynamics_penalty_mean | 0.000677 | +| rollout_info/dynamics_penalty_std | 0.0206 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4945 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5222 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5251 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5228 +num rollout transitions: 50000, reward mean: 4.5275 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.4789 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.5136 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 979 | +| eval/episode_length_std | 67.6 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 9.5 | +| loss/actor | -401 | +| loss/alpha | -0.00234 | +| loss/critic | 44.9 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 2.21 | +| rollout_info/dynamics_penalty_mean | 0.000759 | +| rollout_info/dynamics_penalty_std | 0.0219 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5292 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5306 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5159 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 857 | +| eval/episode_length_std | 320 | +| eval/normalized_episode_reward | 89.7 | +| eval/normalized_episode_reward_std | 35.6 | +| loss/actor | -406 | +| loss/alpha | -0.0159 | +| loss/critic | 41 | +| penalty/lcb_mean | 1.25 | +| penalty/lcb_std | 2.12 | +| rollout_info/dynamics_penalty_mean | 0.000688 | +| rollout_info/dynamics_penalty_std | 0.0206 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5223 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5330 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4792 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5260 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4751 +num rollout transitions: 50000, reward mean: 4.4991 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 992 | +| eval/episode_length_std | 54.1 | +| eval/normalized_episode_reward | 103 | +| eval/normalized_episode_reward_std | 6.21 | +| loss/actor | -404 | +| loss/alpha | 0.00533 | +| loss/critic | 46.9 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 0.000676 | +| rollout_info/dynamics_penalty_std | 0.0205 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4790 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.5266 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4811 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5219 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5125 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.568 | +| loss/actor | -400 | +| loss/alpha | 0.00556 | +| loss/critic | 45.1 | +| penalty/lcb_mean | 1.3 | +| penalty/lcb_std | 2.24 | +| rollout_info/dynamics_penalty_mean | 0.000612 | +| rollout_info/dynamics_penalty_std | 0.0196 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5260 +num rollout transitions: 50000, reward mean: 4.5352 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5264 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.4829 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5246 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5233 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5326 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5308 +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5193 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 0.596 | +| loss/actor | -399 | +| loss/alpha | -0.00524 | +| loss/critic | 45.9 | +| penalty/lcb_mean | 1.31 | +| penalty/lcb_std | 2.28 | +| rollout_info/dynamics_penalty_mean | 0.000611 | +| rollout_info/dynamics_penalty_std | 0.0196 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5201 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.4859 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5164 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5211 +num rollout transitions: 50000, reward mean: 4.5316 +num rollout transitions: 50000, reward mean: 4.5353 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5251 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5122 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 995 | +| eval/episode_length_std | 33.3 | +| eval/normalized_episode_reward | 107 | +| eval/normalized_episode_reward_std | 5.64 | +| loss/actor | -396 | +| loss/alpha | 0.00177 | +| loss/critic | 45.9 | +| penalty/lcb_mean | 1.33 | +| penalty/lcb_std | 2.29 | +| rollout_info/dynamics_penalty_mean | 0.000704 | +| rollout_info/dynamics_penalty_std | 0.0212 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5416 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5172 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5257 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5206 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5219 +num rollout transitions: 50000, reward mean: 4.5217 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4932 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5026 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 99.3 | +| eval/normalized_episode_reward_std | 0.998 | +| loss/actor | -396 | +| loss/alpha | 0.00193 | +| loss/critic | 46.8 | +| penalty/lcb_mean | 1.35 | +| penalty/lcb_std | 2.29 | +| rollout_info/dynamics_penalty_mean | 0.000682 | +| rollout_info/dynamics_penalty_std | 0.0209 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5310 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5268 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5232 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.5210 +---------------------------------------------------------------------------------- +| alpha | 0.136 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 94.9 | +| eval/normalized_episode_reward_std | 0.281 | +| loss/actor | -396 | +| loss/alpha | 0.0184 | +| loss/critic | 49.9 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.35 | +| rollout_info/dynamics_penalty_mean | 0.000694 | +| rollout_info/dynamics_penalty_std | 0.021 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.4891 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4801 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4847 +num rollout transitions: 50000, reward mean: 4.5060 +---------------------------------------------------------------------------------- +| alpha | 0.137 | +| eval/episode_length | 885 | +| eval/episode_length_std | 200 | +| eval/normalized_episode_reward | 88.3 | +| eval/normalized_episode_reward_std | 20.9 | +| loss/actor | -394 | +| loss/alpha | -0.0139 | +| loss/critic | 52.2 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0.000672 | +| rollout_info/dynamics_penalty_std | 0.0208 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4860 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4802 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5237 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.4844 +num rollout transitions: 50000, reward mean: 4.5021 +---------------------------------------------------------------------------------- +| alpha | 0.135 | +| eval/episode_length | 977 | +| eval/episode_length_std | 84.8 | +| eval/normalized_episode_reward | 99.3 | +| eval/normalized_episode_reward_std | 9.68 | +| loss/actor | -393 | +| loss/alpha | 4.8e-05 | +| loss/critic | 52.1 | +| penalty/lcb_mean | 1.42 | +| penalty/lcb_std | 2.42 | +| rollout_info/dynamics_penalty_mean | 0.000609 | +| rollout_info/dynamics_penalty_std | 0.0196 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4834 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4776 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5325 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5053 +---------------------------------------------------------------------------------- +| alpha | 0.133 | +| eval/episode_length | 912 | +| eval/episode_length_std | 189 | +| eval/normalized_episode_reward | 92.2 | +| eval/normalized_episode_reward_std | 21.4 | +| loss/actor | -394 | +| loss/alpha | 0.00535 | +| loss/critic | 49.9 | +| penalty/lcb_mean | 1.39 | +| penalty/lcb_std | 2.38 | +| rollout_info/dynamics_penalty_mean | 0.000627 | +| rollout_info/dynamics_penalty_std | 0.02 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4822 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4806 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.4843 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4773 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.5112 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 714 | +| eval/episode_length_std | 248 | +| eval/normalized_episode_reward | 70.9 | +| eval/normalized_episode_reward_std | 27.8 | +| loss/actor | -395 | +| loss/alpha | -0.00814 | +| loss/critic | 51.4 | +| penalty/lcb_mean | 1.41 | +| penalty/lcb_std | 2.46 | +| rollout_info/dynamics_penalty_mean | 0.000637 | +| rollout_info/dynamics_penalty_std | 0.0202 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4788 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4820 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4872 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.4914 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4902 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4854 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4890 +---------------------------------------------------------------------------------- +| alpha | 0.14 | +| eval/episode_length | 658 | +| eval/episode_length_std | 257 | +| eval/normalized_episode_reward | 64.7 | +| eval/normalized_episode_reward_std | 27.6 | +| loss/actor | -392 | +| loss/alpha | 0.00556 | +| loss/critic | 54.6 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.51 | +| rollout_info/dynamics_penalty_mean | 0.000601 | +| rollout_info/dynamics_penalty_std | 0.0195 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4926 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4831 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.4848 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4876 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4815 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5259 +num rollout transitions: 50000, reward mean: 4.5020 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4851 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4908 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5076 +---------------------------------------------------------------------------------- +| alpha | 0.137 | +| eval/episode_length | 974 | +| eval/episode_length_std | 110 | +| eval/normalized_episode_reward | 101 | +| eval/normalized_episode_reward_std | 13.4 | +| loss/actor | -391 | +| loss/alpha | -0.00827 | +| loss/critic | 55.5 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.55 | +| rollout_info/dynamics_penalty_mean | 0.000576 | +| rollout_info/dynamics_penalty_std | 0.019 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5224 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4805 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5258 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5179 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4849 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.4980 +---------------------------------------------------------------------------------- +| alpha | 0.136 | +| eval/episode_length | 988 | +| eval/episode_length_std | 61.4 | +| eval/normalized_episode_reward | 104 | +| eval/normalized_episode_reward_std | 7.18 | +| loss/actor | -393 | +| loss/alpha | 0.0147 | +| loss/critic | 55.1 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.55 | +| rollout_info/dynamics_penalty_mean | 0.000589 | +| rollout_info/dynamics_penalty_std | 0.0192 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4801 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5354 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5210 +---------------------------------------------------------------------------------- +| alpha | 0.137 | +| eval/episode_length | 959 | +| eval/episode_length_std | 114 | +| eval/normalized_episode_reward | 98.5 | +| eval/normalized_episode_reward_std | 13.2 | +| loss/actor | -393 | +| loss/alpha | -0.00275 | +| loss/critic | 58 | +| penalty/lcb_mean | 1.52 | +| penalty/lcb_std | 2.61 | +| rollout_info/dynamics_penalty_mean | 0.000681 | +| rollout_info/dynamics_penalty_std | 0.0209 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5234 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.4802 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5161 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5350 +num rollout transitions: 50000, reward mean: 4.4701 +num rollout transitions: 50000, reward mean: 4.4662 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4834 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 999 | +| eval/episode_length_std | 5.17 | +| eval/normalized_episode_reward | 98.2 | +| eval/normalized_episode_reward_std | 3.28 | +| loss/actor | -398 | +| loss/alpha | -0.0101 | +| loss/critic | 55.7 | +| penalty/lcb_mean | 1.49 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0.000604 | +| rollout_info/dynamics_penalty_std | 0.0193 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.4852 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4804 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.4881 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5254 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.4864 +num rollout transitions: 50000, reward mean: 4.4916 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5100 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 75 | +| eval/normalized_episode_reward_std | 14.1 | +| loss/actor | -395 | +| loss/alpha | 0.0152 | +| loss/critic | 57.7 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.000685 | +| rollout_info/dynamics_penalty_std | 0.0209 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5219 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.4842 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5265 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.5243 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4956 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5199 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4890 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5253 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 2.66 | +| loss/actor | -393 | +| loss/alpha | -0.00879 | +| loss/critic | 53.4 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.51 | +| rollout_info/dynamics_penalty_mean | 0.000642 | +| rollout_info/dynamics_penalty_std | 0.0201 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5285 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5241 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4778 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.5040 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 3.11 | +| loss/actor | -395 | +| loss/alpha | -0.00932 | +| loss/critic | 54.4 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0.000626 | +| rollout_info/dynamics_penalty_std | 0.0198 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5341 +num rollout transitions: 50000, reward mean: 4.5126 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5276 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4824 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4772 +num rollout transitions: 50000, reward mean: 4.4901 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5169 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5198 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.5084 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 984 | +| eval/episode_length_std | 67.3 | +| eval/normalized_episode_reward | 111 | +| eval/normalized_episode_reward_std | 8.15 | +| loss/actor | -396 | +| loss/alpha | 0.0114 | +| loss/critic | 54 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.53 | +| rollout_info/dynamics_penalty_mean | 0.000532 | +| rollout_info/dynamics_penalty_std | 0.0181 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.5257 +num rollout transitions: 50000, reward mean: 4.5260 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5247 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5150 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.5215 +num rollout transitions: 50000, reward mean: 4.5280 +num rollout transitions: 50000, reward mean: 4.5210 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5156 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.5066 +---------------------------------------------------------------------------------- +| alpha | 0.129 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 106 | +| eval/normalized_episode_reward_std | 1.57 | +| loss/actor | -398 | +| loss/alpha | -0.0188 | +| loss/critic | 54.3 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.5 | +| rollout_info/dynamics_penalty_mean | 0.000606 | +| rollout_info/dynamics_penalty_std | 0.0196 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4870 +num rollout transitions: 50000, reward mean: 4.4862 +num rollout transitions: 50000, reward mean: 4.4815 +num rollout transitions: 50000, reward mean: 4.5045 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5190 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5089 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4928 +num rollout transitions: 50000, reward mean: 4.4865 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5141 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 1.56 | +| loss/actor | -398 | +| loss/alpha | 0.00896 | +| loss/critic | 53.9 | +| penalty/lcb_mean | 1.46 | +| penalty/lcb_std | 2.51 | +| rollout_info/dynamics_penalty_mean | 0.00063 | +| rollout_info/dynamics_penalty_std | 0.0201 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 1900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5251 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5363 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5124 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 0.258 | +| loss/actor | -394 | +| loss/alpha | 0.0013 | +| loss/critic | 50.8 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.43 | +| rollout_info/dynamics_penalty_mean | 0.000565 | +| rollout_info/dynamics_penalty_std | 0.0189 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 1950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4878 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4833 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4855 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5225 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5166 +num rollout transitions: 50000, reward mean: 4.5258 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.4868 +num rollout transitions: 50000, reward mean: 4.4981 +---------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 996 | +| eval/episode_length_std | 42.1 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 5.09 | +| loss/actor | -396 | +| loss/alpha | -0.00378 | +| loss/critic | 52.6 | +| penalty/lcb_mean | 1.44 | +| penalty/lcb_std | 2.47 | +| rollout_info/dynamics_penalty_mean | 0.000559 | +| rollout_info/dynamics_penalty_std | 0.0187 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2000000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5104 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5229 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5204 +num rollout transitions: 50000, reward mean: 4.5269 +num rollout transitions: 50000, reward mean: 4.5167 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4774 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5081 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5296 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4921 +num rollout transitions: 50000, reward mean: 4.4927 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.719 | +| loss/actor | -393 | +| loss/alpha | -0.00869 | +| loss/critic | 53 | +| penalty/lcb_mean | 1.47 | +| penalty/lcb_std | 2.48 | +| rollout_info/dynamics_penalty_mean | 0.000531 | +| rollout_info/dynamics_penalty_std | 0.0183 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2050000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5162 +num rollout transitions: 50000, reward mean: 4.5181 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5245 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5218 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5197 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5120 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5176 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5168 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5304 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5072 +---------------------------------------------------------------------------------- +| alpha | 0.131 | +| eval/episode_length | 999 | +| eval/episode_length_std | 8.86 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 2.33 | +| loss/actor | -392 | +| loss/alpha | 0.0198 | +| loss/critic | 57.3 | +| penalty/lcb_mean | 1.53 | +| penalty/lcb_std | 2.56 | +| rollout_info/dynamics_penalty_mean | 0.000523 | +| rollout_info/dynamics_penalty_std | 0.0181 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2100000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5123 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4822 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5109 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5144 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5057 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.4890 +---------------------------------------------------------------------------------- +| alpha | 0.13 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 0.248 | +| loss/actor | -393 | +| loss/alpha | -0.0125 | +| loss/critic | 55.7 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.57 | +| rollout_info/dynamics_penalty_mean | 0.000575 | +| rollout_info/dynamics_penalty_std | 0.0191 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2150000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5240 +num rollout transitions: 50000, reward mean: 4.5303 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4830 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5148 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5235 +num rollout transitions: 50000, reward mean: 4.5317 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5188 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4896 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5034 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 102 | +| eval/normalized_episode_reward_std | 1.24 | +| loss/actor | -394 | +| loss/alpha | 0.00471 | +| loss/critic | 55 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0.000575 | +| rollout_info/dynamics_penalty_std | 0.019 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2200000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.4942 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.5157 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5350 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5216 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5175 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5193 +num rollout transitions: 50000, reward mean: 4.5231 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5263 +---------------------------------------------------------------------------------- +| alpha | 0.12 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 109 | +| eval/normalized_episode_reward_std | 0.331 | +| loss/actor | -396 | +| loss/alpha | 0.00845 | +| loss/critic | 55 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.56 | +| rollout_info/dynamics_penalty_mean | 0.000664 | +| rollout_info/dynamics_penalty_std | 0.0207 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2250000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.4869 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5098 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5191 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5187 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4816 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5143 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.4991 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5040 +---------------------------------------------------------------------------------- +| alpha | 0.134 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 108 | +| eval/normalized_episode_reward_std | 2.63 | +| loss/actor | -393 | +| loss/alpha | 0.00271 | +| loss/critic | 59 | +| penalty/lcb_mean | 1.56 | +| penalty/lcb_std | 2.64 | +| rollout_info/dynamics_penalty_mean | 0.000565 | +| rollout_info/dynamics_penalty_std | 0.0189 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2300000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4939 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.4924 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5072 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5025 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.5182 +num rollout transitions: 50000, reward mean: 4.5209 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5196 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4818 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4858 +num rollout transitions: 50000, reward mean: 4.4877 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4880 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4995 +num rollout transitions: 50000, reward mean: 4.5029 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 1.57 | +| loss/actor | -393 | +| loss/alpha | -0.011 | +| loss/critic | 58.2 | +| penalty/lcb_mean | 1.51 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.000583 | +| rollout_info/dynamics_penalty_std | 0.0193 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2350000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.4911 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5119 +num rollout transitions: 50000, reward mean: 4.5250 +num rollout transitions: 50000, reward mean: 4.5142 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4959 +num rollout transitions: 50000, reward mean: 4.4974 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4913 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5252 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5152 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5092 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 116 | +| eval/normalized_episode_reward_std | 0.885 | +| loss/actor | -393 | +| loss/alpha | 0.00244 | +| loss/critic | 55 | +| penalty/lcb_mean | 1.48 | +| penalty/lcb_std | 2.54 | +| rollout_info/dynamics_penalty_mean | 0.000692 | +| rollout_info/dynamics_penalty_std | 0.021 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2400000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.4746 +num rollout transitions: 50000, reward mean: 4.4899 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.4837 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5240 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.4861 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.4953 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4850 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.5029 +---------------------------------------------------------------------------------- +| alpha | 0.123 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 1.82 | +| loss/actor | -394 | +| loss/alpha | 0.00363 | +| loss/critic | 56.6 | +| penalty/lcb_mean | 1.5 | +| penalty/lcb_std | 2.59 | +| rollout_info/dynamics_penalty_mean | 0.000699 | +| rollout_info/dynamics_penalty_std | 0.0213 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2450000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5200 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5146 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.5009 +num rollout transitions: 50000, reward mean: 4.4981 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5236 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5118 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5177 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.5124 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.5171 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.4995 +---------------------------------------------------------------------------------- +| alpha | 0.132 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 113 | +| eval/normalized_episode_reward_std | 0.855 | +| loss/actor | -389 | +| loss/alpha | 0.00578 | +| loss/critic | 59.8 | +| penalty/lcb_mean | 1.56 | +| penalty/lcb_std | 2.62 | +| rollout_info/dynamics_penalty_mean | 0.000662 | +| rollout_info/dynamics_penalty_std | 0.0206 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2500000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5096 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4887 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5076 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5102 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5023 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4978 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4969 +num rollout transitions: 50000, reward mean: 4.4822 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.4889 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5115 +num rollout transitions: 50000, reward mean: 4.4863 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.5173 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5097 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4803 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5158 +num rollout transitions: 50000, reward mean: 4.4931 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5003 +---------------------------------------------------------------------------------- +| alpha | 0.127 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 114 | +| eval/normalized_episode_reward_std | 1.93 | +| loss/actor | -391 | +| loss/alpha | -0.00142 | +| loss/critic | 59.7 | +| penalty/lcb_mean | 1.56 | +| penalty/lcb_std | 2.63 | +| rollout_info/dynamics_penalty_mean | 0.000688 | +| rollout_info/dynamics_penalty_std | 0.021 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2550000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5178 +num rollout transitions: 50000, reward mean: 4.5139 +num rollout transitions: 50000, reward mean: 4.5125 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5131 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4935 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4839 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.4919 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4845 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.5145 +num rollout transitions: 50000, reward mean: 4.4903 +num rollout transitions: 50000, reward mean: 4.5165 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5185 +num rollout transitions: 50000, reward mean: 4.5094 +num rollout transitions: 50000, reward mean: 4.5163 +num rollout transitions: 50000, reward mean: 4.5111 +num rollout transitions: 50000, reward mean: 4.5105 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4999 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5189 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5068 +num rollout transitions: 50000, reward mean: 4.5239 +num rollout transitions: 50000, reward mean: 4.5273 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.5002 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4892 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 997 | +| eval/episode_length_std | 25.9 | +| eval/normalized_episode_reward | 112 | +| eval/normalized_episode_reward_std | 3.73 | +| loss/actor | -392 | +| loss/alpha | -0.0223 | +| loss/critic | 59.5 | +| penalty/lcb_mean | 1.54 | +| penalty/lcb_std | 2.65 | +| rollout_info/dynamics_penalty_mean | 0.000651 | +| rollout_info/dynamics_penalty_std | 0.0204 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2600000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4873 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.5212 +num rollout transitions: 50000, reward mean: 4.5153 +num rollout transitions: 50000, reward mean: 4.5114 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.4897 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4802 +num rollout transitions: 50000, reward mean: 4.4912 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.4871 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5080 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5192 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5049 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.4904 +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4989 +num rollout transitions: 50000, reward mean: 4.5083 +num rollout transitions: 50000, reward mean: 4.5063 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4828 +num rollout transitions: 50000, reward mean: 4.4893 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4836 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.5016 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.4831 +num rollout transitions: 50000, reward mean: 4.4987 +---------------------------------------------------------------------------------- +| alpha | 0.128 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 118 | +| eval/normalized_episode_reward_std | 0.742 | +| loss/actor | -391 | +| loss/alpha | 0.0211 | +| loss/critic | 65 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.76 | +| rollout_info/dynamics_penalty_mean | 0.000591 | +| rollout_info/dynamics_penalty_std | 0.0193 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2650000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5121 +num rollout transitions: 50000, reward mean: 4.5095 +num rollout transitions: 50000, reward mean: 4.5170 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4993 +num rollout transitions: 50000, reward mean: 4.5132 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4990 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.5122 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5032 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5154 +num rollout transitions: 50000, reward mean: 4.5013 +num rollout transitions: 50000, reward mean: 4.4961 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5138 +num rollout transitions: 50000, reward mean: 4.4883 +num rollout transitions: 50000, reward mean: 4.5183 +num rollout transitions: 50000, reward mean: 4.5174 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5180 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5112 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4943 +num rollout transitions: 50000, reward mean: 4.4764 +num rollout transitions: 50000, reward mean: 4.4925 +num rollout transitions: 50000, reward mean: 4.4930 +num rollout transitions: 50000, reward mean: 4.4917 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5069 +num rollout transitions: 50000, reward mean: 4.5113 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5082 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4915 +num rollout transitions: 50000, reward mean: 4.4998 +num rollout transitions: 50000, reward mean: 4.5018 +---------------------------------------------------------------------------------- +| alpha | 0.125 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 110 | +| eval/normalized_episode_reward_std | 0.539 | +| loss/actor | -394 | +| loss/alpha | 0.00459 | +| loss/critic | 64.4 | +| penalty/lcb_mean | 1.62 | +| penalty/lcb_std | 2.75 | +| rollout_info/dynamics_penalty_mean | 0.000588 | +| rollout_info/dynamics_penalty_std | 0.0194 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2700000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4946 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4941 +num rollout transitions: 50000, reward mean: 4.4922 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5093 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.4962 +num rollout transitions: 50000, reward mean: 4.5184 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5065 +num rollout transitions: 50000, reward mean: 4.4952 +num rollout transitions: 50000, reward mean: 4.5007 +num rollout transitions: 50000, reward mean: 4.4905 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5062 +num rollout transitions: 50000, reward mean: 4.5056 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5262 +num rollout transitions: 50000, reward mean: 4.5159 +num rollout transitions: 50000, reward mean: 4.5130 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4879 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5088 +num rollout transitions: 50000, reward mean: 4.5110 +num rollout transitions: 50000, reward mean: 4.5140 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.4929 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.5079 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5136 +num rollout transitions: 50000, reward mean: 4.5127 +num rollout transitions: 50000, reward mean: 4.5067 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.5070 +---------------------------------------------------------------------------------- +| alpha | 0.121 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 119 | +| eval/normalized_episode_reward_std | 1.77 | +| loss/actor | -393 | +| loss/alpha | -0.0115 | +| loss/critic | 62.4 | +| penalty/lcb_mean | 1.59 | +| penalty/lcb_std | 2.72 | +| rollout_info/dynamics_penalty_mean | 0.000595 | +| rollout_info/dynamics_penalty_std | 0.0193 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2750000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5141 +num rollout transitions: 50000, reward mean: 4.5137 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5220 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5155 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5149 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4895 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.5012 +num rollout transitions: 50000, reward mean: 4.5028 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5195 +num rollout transitions: 50000, reward mean: 4.5116 +num rollout transitions: 50000, reward mean: 4.5238 +num rollout transitions: 50000, reward mean: 4.5058 +num rollout transitions: 50000, reward mean: 4.5134 +num rollout transitions: 50000, reward mean: 4.5186 +num rollout transitions: 50000, reward mean: 4.5160 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4960 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4977 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5071 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.5103 +num rollout transitions: 50000, reward mean: 4.5107 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4940 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5151 +num rollout transitions: 50000, reward mean: 4.5074 +---------------------------------------------------------------------------------- +| alpha | 0.122 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 115 | +| eval/normalized_episode_reward_std | 2.01 | +| loss/actor | -393 | +| loss/alpha | 0.0223 | +| loss/critic | 63.8 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.78 | +| rollout_info/dynamics_penalty_mean | 0.000615 | +| rollout_info/dynamics_penalty_std | 0.0197 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.51 | +| timestep | 2800000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4997 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.5035 +num rollout transitions: 50000, reward mean: 4.4910 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.4987 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4900 +num rollout transitions: 50000, reward mean: 4.4949 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5194 +num rollout transitions: 50000, reward mean: 4.4886 +num rollout transitions: 50000, reward mean: 4.4936 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.4988 +num rollout transitions: 50000, reward mean: 4.5064 +num rollout transitions: 50000, reward mean: 4.5085 +num rollout transitions: 50000, reward mean: 4.4982 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5147 +num rollout transitions: 50000, reward mean: 4.5039 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5084 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4906 +num rollout transitions: 50000, reward mean: 4.4958 +num rollout transitions: 50000, reward mean: 4.4846 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.4898 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.5051 +num rollout transitions: 50000, reward mean: 4.4983 +num rollout transitions: 50000, reward mean: 4.4944 +num rollout transitions: 50000, reward mean: 4.5038 +num rollout transitions: 50000, reward mean: 4.4841 +num rollout transitions: 50000, reward mean: 4.5074 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4918 +num rollout transitions: 50000, reward mean: 4.4885 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4985 +num rollout transitions: 50000, reward mean: 4.4949 +---------------------------------------------------------------------------------- +| alpha | 0.126 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 120 | +| eval/normalized_episode_reward_std | 1.7 | +| loss/actor | -393 | +| loss/alpha | -0.00776 | +| loss/critic | 67.2 | +| penalty/lcb_mean | 1.64 | +| penalty/lcb_std | 2.8 | +| rollout_info/dynamics_penalty_mean | 0.000614 | +| rollout_info/dynamics_penalty_std | 0.0197 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2850000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4827 +num rollout transitions: 50000, reward mean: 4.4948 +num rollout transitions: 50000, reward mean: 4.4882 +num rollout transitions: 50000, reward mean: 4.4972 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5117 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.4980 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.5059 +num rollout transitions: 50000, reward mean: 4.5054 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5044 +num rollout transitions: 50000, reward mean: 4.5100 +num rollout transitions: 50000, reward mean: 4.5090 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.5087 +num rollout transitions: 50000, reward mean: 4.5086 +num rollout transitions: 50000, reward mean: 4.4954 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.5053 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.4979 +num rollout transitions: 50000, reward mean: 4.4894 +num rollout transitions: 50000, reward mean: 4.4933 +num rollout transitions: 50000, reward mean: 4.5037 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5092 +num rollout transitions: 50000, reward mean: 4.4923 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.4951 +num rollout transitions: 50000, reward mean: 4.4816 +num rollout transitions: 50000, reward mean: 4.4838 +num rollout transitions: 50000, reward mean: 4.4813 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5036 +num rollout transitions: 50000, reward mean: 4.5060 +num rollout transitions: 50000, reward mean: 4.5046 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5129 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4965 +---------------------------------------------------------------------------------- +| alpha | 0.117 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 122 | +| eval/normalized_episode_reward_std | 1.29 | +| loss/actor | -392 | +| loss/alpha | -0.00297 | +| loss/critic | 64 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.77 | +| rollout_info/dynamics_penalty_mean | 0.000621 | +| rollout_info/dynamics_penalty_std | 0.0199 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2900000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.4976 +num rollout transitions: 50000, reward mean: 4.4970 +num rollout transitions: 50000, reward mean: 4.5031 +num rollout transitions: 50000, reward mean: 4.4957 +num rollout transitions: 50000, reward mean: 4.4963 +num rollout transitions: 50000, reward mean: 4.5108 +num rollout transitions: 50000, reward mean: 4.5135 +num rollout transitions: 50000, reward mean: 4.4992 +num rollout transitions: 50000, reward mean: 4.4973 +num rollout transitions: 50000, reward mean: 4.4966 +num rollout transitions: 50000, reward mean: 4.5050 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5073 +num rollout transitions: 50000, reward mean: 4.5055 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5041 +num rollout transitions: 50000, reward mean: 4.5024 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.5033 +num rollout transitions: 50000, reward mean: 4.4927 +num rollout transitions: 50000, reward mean: 4.4884 +num rollout transitions: 50000, reward mean: 4.5011 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5099 +num rollout transitions: 50000, reward mean: 4.5106 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5077 +num rollout transitions: 50000, reward mean: 4.5021 +num rollout transitions: 50000, reward mean: 4.5022 +num rollout transitions: 50000, reward mean: 4.5075 +num rollout transitions: 50000, reward mean: 4.5042 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5043 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5133 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5128 +num rollout transitions: 50000, reward mean: 4.5010 +num rollout transitions: 50000, reward mean: 4.5026 +num rollout transitions: 50000, reward mean: 4.5101 +num rollout transitions: 50000, reward mean: 4.4920 +num rollout transitions: 50000, reward mean: 4.5005 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5030 +num rollout transitions: 50000, reward mean: 4.5045 +---------------------------------------------------------------------------------- +| alpha | 0.113 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 122 | +| eval/normalized_episode_reward_std | 1.78 | +| loss/actor | -384 | +| loss/alpha | -0.032 | +| loss/critic | 64 | +| penalty/lcb_mean | 1.6 | +| penalty/lcb_std | 2.75 | +| rollout_info/dynamics_penalty_mean | 0.000594 | +| rollout_info/dynamics_penalty_std | 0.0194 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 2950000 | +---------------------------------------------------------------------------------- +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4907 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.5018 +num rollout transitions: 50000, reward mean: 4.5019 +num rollout transitions: 50000, reward mean: 4.4984 +num rollout transitions: 50000, reward mean: 4.4955 +num rollout transitions: 50000, reward mean: 4.5001 +num rollout transitions: 50000, reward mean: 4.5017 +num rollout transitions: 50000, reward mean: 4.5029 +num rollout transitions: 50000, reward mean: 4.5003 +num rollout transitions: 50000, reward mean: 4.4975 +num rollout transitions: 50000, reward mean: 4.5078 +num rollout transitions: 50000, reward mean: 4.4971 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4892 +num rollout transitions: 50000, reward mean: 4.4986 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5070 +num rollout transitions: 50000, reward mean: 4.5027 +num rollout transitions: 50000, reward mean: 4.5014 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4994 +num rollout transitions: 50000, reward mean: 4.4938 +num rollout transitions: 50000, reward mean: 4.4934 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.4968 +num rollout transitions: 50000, reward mean: 4.4965 +num rollout transitions: 50000, reward mean: 4.4866 +num rollout transitions: 50000, reward mean: 4.5091 +num rollout transitions: 50000, reward mean: 4.4937 +num rollout transitions: 50000, reward mean: 4.5061 +num rollout transitions: 50000, reward mean: 4.5000 +num rollout transitions: 50000, reward mean: 4.5047 +num rollout transitions: 50000, reward mean: 4.5034 +num rollout transitions: 50000, reward mean: 4.4909 +num rollout transitions: 50000, reward mean: 4.5048 +num rollout transitions: 50000, reward mean: 4.5004 +num rollout transitions: 50000, reward mean: 4.5006 +num rollout transitions: 50000, reward mean: 4.4950 +num rollout transitions: 50000, reward mean: 4.4964 +num rollout transitions: 50000, reward mean: 4.5066 +num rollout transitions: 50000, reward mean: 4.4967 +num rollout transitions: 50000, reward mean: 4.5008 +num rollout transitions: 50000, reward mean: 4.4996 +num rollout transitions: 50000, reward mean: 4.4947 +num rollout transitions: 50000, reward mean: 4.5052 +num rollout transitions: 50000, reward mean: 4.5015 +num rollout transitions: 50000, reward mean: 4.5040 +num rollout transitions: 50000, reward mean: 4.4949 +---------------------------------------------------------------------------------- +| alpha | 0.138 | +| eval/episode_length | 1e+03 | +| eval/episode_length_std | 0 | +| eval/normalized_episode_reward | 120 | +| eval/normalized_episode_reward_std | 1.56 | +| loss/actor | -384 | +| loss/alpha | 0.0645 | +| loss/critic | 63.9 | +| penalty/lcb_mean | 1.61 | +| penalty/lcb_std | 2.75 | +| rollout_info/dynamics_penalty_mean | 0.000555 | +| rollout_info/dynamics_penalty_std | 0.0187 | +| rollout_info/num_transitions | 5e+04 | +| rollout_info/reward_mean | 4.5 | +| timestep | 3000000 | +---------------------------------------------------------------------------------- +total time: 140058.04s diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-050351/record/dynamics_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-050351/record/dynamics_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-050351/record/hyper_param.json b/walker2d_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-050351/record/hyper_param.json new file mode 100644 index 0000000000000000000000000000000000000000..78e69ce0d98b8a9b0f3ad30edcce48a24b338403 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-050351/record/hyper_param.json @@ -0,0 +1 @@ +{"config": "config_gormpo/walker2d_medium_expert_72.5_vae.yaml", "algo_name": "mobile_gormpo", "task": "walker2d-medium-expert-v2", "seed": 456, "device": "cuda:4", "dataset_path": "/public/d4rl/sparse_datasets/walker2d_medium_expert_sparse_73.pkl", "dynamics_path": "log/walker2d-medium-expert-v2/seed_456×tamp_26-0112-030559/model", "reward_penalty_coef": 0.7, "classifier_model_name": "/public/gormpo/models/walker2d_medium_expert_sparse_3/vae", "use_wandb": true, "wandb_project": "mobile-gormpo", "wandb_entity": null, "domain": "gym", "actor_lr": 0.0001, "critic_lr": 0.0003, "hidden_dims": [256, 256], "gamma": 0.99, "tau": 0.005, "alpha": 0.2, "auto_alpha": true, "target_entropy": -6.0, "alpha_lr": 0.0001, "deterministic_backup": true, "max_q_backup": false, "norm_reward": false, "num_q_ensemble": 2, "dynamics_lr": 0.001, "dynamics_max_epochs": null, "max_epochs_since_update": 5, "dynamics_hidden_dims": [200, 200, 200, 200], "dynamics_weight_decay": [2.5e-05, 5e-05, 7.5e-05, 7.5e-05, 0.0001], "n_ensemble": 7, "n_elites": 5, "rollout_freq": 1000, "rollout_batch_size": 50000, "rollout_length": 1, "penalty_coef": 1.5, "num_samples": 10, "model_retain_epochs": 5, "real_ratio": 0.05, "load_dynamics_path": null, "epoch": 3000, "step_per_epoch": 1000, "eval_episodes": 100, "batch_size": 256, "lr_scheduler": true, "penalty_type": "tanh", "density_model": "vae", "target_dim": 23, "obs_shape": [17], "action_dim": 6.0, "max_action": 1.0, "classifier_type": "vae"} diff --git a/walker2d_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-050351/record/policy_training_progress.csv b/walker2d_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-050351/record/policy_training_progress.csv new file mode 100644 index 0000000000000000000000000000000000000000..7334e817c6c581d4cd872a4916c58a773562d302 --- /dev/null +++ b/walker2d_medium_expert_sparse/mobile_based/vae/seed_456×tamp_26-0320-050351/record/policy_training_progress.csv @@ -0,0 +1,61 @@ +alpha,eval/episode_length,eval/episode_length_std,eval/normalized_episode_reward,eval/normalized_episode_reward_std,loss/actor,loss/alpha,loss/critic,penalty/lcb_mean,penalty/lcb_std,rollout_info/dynamics_penalty_mean,rollout_info/dynamics_penalty_std,rollout_info/num_transitions,rollout_info/reward_mean,timestep +0.2517500447058655,387.04,259.6463718213679,10.292420473662151,7.69684128997323,-287.94223908371606,-1.7698135237266068,17.95338806394184,0.9140278797458956,1.1759110702655375,0.0019564669649430873,0.030471530507896978,50000.0,4.525452872375484,50000 +0.1295945608718689,188.69,61.10166855332185,8.824598062577078,6.124819885006266,-437.6233095751939,0.008867046581916279,28.37817338014563,1.1335428318679308,1.6569103817093205,0.0011691094891457352,0.02798095448396504,50000.0,4.502245108108536,100000 +0.12040116908848218,295.23,161.728096198527,25.934606973089146,18.031987125551822,-454.2866212756349,-0.014619725197888856,23.997251048526692,1.0502361711942978,1.5710068039226674,0.0010965254924291403,0.027117629980260075,50000.0,4.50726862893923,150000 +0.1242937356646362,396.08,199.66074626726206,30.69339167488553,21.420516554771453,-449.68548230590426,0.01695535412628192,25.53606681259193,1.0723598176193319,1.5839747586834518,0.0009215029620562158,0.0247858699310395,50000.0,4.519823466503297,200000 +0.13657554961592241,763.27,274.68403138879404,67.32092786472406,27.662667324844442,-435.6479435131813,0.01431470453090974,30.696055280618882,1.1442134934198815,1.7194558438766165,0.0007140793061267176,0.02142697446237388,50000.0,4.518209619171469,250000 +0.15592887935578464,192.79,177.12212143038485,11.642294453115547,17.024761718263907,-426.8484185638512,-0.0009495853239484432,38.60730372205778,1.2863478972017997,1.9238722545111253,0.0006976838488014986,0.020957706112381756,50000.0,4.514679369907907,300000 +0.15515002270221356,903.37,265.24700394160914,87.62156732945276,27.180263990970577,-414.1837761108417,0.0037073743430524084,36.54779146917309,1.2584651341235682,1.8334567161369324,0.0007084946619266791,0.021373426170119655,50000.0,4.5116005039887614,350000 +0.1664459681886476,844.65,272.99125901757367,82.54064905439157,28.77764850603166,-399.80681448059516,0.009852887674271988,41.32581809982306,1.3197106144714046,1.9367729581701776,0.0006727904045236272,0.0203544355141676,50000.0,4.510587061086055,400000 +0.17046326831072806,698.49,336.1876111637667,64.14464482608636,33.67448579288255,-397.5982278521724,0.0037592584751917845,46.806836774195894,1.38230726439954,2.0884843834912687,0.0006564540037385888,0.020053621712839566,50000.0,4.511974347814345,450000 +0.17079438675284098,912.31,141.78192374206242,93.62905116627451,15.450731395538678,-396.36180283630546,-0.014359542279578787,47.51919209905632,1.3963742569661073,2.1358845799707917,0.0007012040322085493,0.020919470884384367,50000.0,4.508742586906833,500000 +0.16277012547254374,848.37,250.62213210329207,80.55122523742465,25.70030366106507,-394.34732590820505,-0.0026443739466368112,45.78840823228858,1.3655736074602565,2.1195679858827443,0.0007128827636860516,0.021075483592492412,50000.0,4.51246236139743,550000 +0.15432852597296243,849.54,169.92518471374396,86.28783305406085,19.222826232969567,-396.5437843920913,-0.004572401590459107,45.380565103072634,1.3525694607400847,2.1450341038132015,0.0007579009899319092,0.021811085255279264,50000.0,4.509148438081497,600000 +0.14888550187111133,1000.0,0.0,104.6103900143633,1.7624036176259739,-399.50881264709597,-0.0018712595238537552,44.917200930919954,1.3296550399648979,2.165636950451116,0.000662308207851928,0.020231210874043958,50000.0,4.512314463434282,650000 +0.14827011271476373,1000.0,0.0,105.2941255310984,1.537018540211588,-400.7652507305915,-0.004781963196955578,45.419990606117516,1.3148482260882706,2.189432968928788,0.0006939997969684427,0.020797579054402313,50000.0,4.510254036889941,700000 +0.14074010889887661,919.34,226.78836919030923,99.36683241796544,26.4514444098295,-402.69126046020295,0.006152659326493726,44.58097090631501,1.2944587580787943,2.187295517114378,0.0006773275071225525,0.02060876564564208,50000.0,4.509936117013654,750000 +0.1395161807641416,978.94,67.63029202953363,102.90767314841209,9.497462498743008,-401.3217674243133,-0.002336362635232419,44.93379861556977,1.3065238029718471,2.208941412909001,0.0007585861446072535,0.021909745066598253,50000.0,4.505357453177307,800000 +0.12695863843932734,856.58,320.33145896087063,89.70111970299322,35.621547810398766,-406.24640975341595,-0.015903903615362404,41.022054960479046,1.2456126704680983,2.1197179602503895,0.0006875716059996354,0.02057552824375395,50000.0,4.506132146724821,850000 +0.12929765757754316,991.86,54.13926855804389,103.40639045934759,6.214470784832885,-404.30654150573395,0.005326374838166111,46.86141012945178,1.3121838390123712,2.2762008755850514,0.0006761630422287328,0.02053798585612069,50000.0,4.505788896680802,900000 +0.13051095973253266,1000.0,0.0,109.02220111682925,0.568465367448134,-400.05453853698947,0.005558404710777037,45.08069673017562,1.2991647856891135,2.2380160940790312,0.0006122496581412151,0.01962830558521832,50000.0,4.505794260392574,950000 +0.13040726691648444,1000.0,0.0,104.4730488917434,0.5956803234721181,-399.38067407592655,-0.005244898038469194,45.90445458082205,1.3135050270128241,2.2771459875142304,0.0006107398609225401,0.019582202992042056,50000.0,4.5102785335844535,1000000 +0.13096426387816462,995.27,33.32142103812502,107.02455854541817,5.641444739251747,-396.07433333618314,0.0017671962143852935,45.92647427001903,1.3280090216398175,2.285706766917709,0.0007043621393515893,0.02121141055133201,50000.0,4.510267466746453,1050000 +0.13448903790503994,1000.0,0.0,99.26428944745909,0.9981897232718868,-396.15153350524946,0.0019288344317675108,46.76188239011724,1.3496203788638106,2.2901654028773333,0.0006820637509191644,0.020905191138791437,50000.0,4.508529419323958,1100000 +0.13561132081180605,1000.0,0.0,94.92907404520811,0.28101143088545927,-396.0292199054022,0.01837606993544845,49.94559993232721,1.4174698106444106,2.3494982786869887,0.0006943515141801677,0.02098369276958063,50000.0,4.505429814361134,1150000 +0.13737850825965442,884.6,200.07578564134144,88.28958940168957,20.905198926036192,-393.605708850707,-0.013887650070339375,52.241249224757766,1.437909578578472,2.4579845376133775,0.0006716677520904139,0.0208403948324507,50000.0,4.5029989675725215,1200000 +0.1351047145722806,976.51,84.78755745980656,99.29930365055667,9.681249198389505,-393.44566710632074,4.801061354563891e-05,52.06883593221715,1.4179739466607566,2.4161890561413775,0.0006089739833338488,0.019609809218048516,50000.0,4.500685658238433,1250000 +0.1326805297724878,912.19,188.6910541069714,92.24669096517583,21.41763651098251,-393.68954500854755,0.005346685126200201,49.85028730506854,1.391536139311759,2.3836714988577463,0.000626614708790961,0.020003632344118522,50000.0,4.50261692913301,1300000 +0.13134290354699088,713.79,247.82390098616395,70.88083242021939,27.801384073475166,-395.2833699743705,-0.008141266186647057,51.41644955822017,1.4142742062020053,2.4601669651210587,0.0006370585799783591,0.02023276349199129,50000.0,4.498473238968048,1350000 +0.13975938896805118,658.45,257.14790977178876,64.69918985011857,27.57751172156628,-392.10155067749054,0.005559992543868474,54.59912068389834,1.4666769309270387,2.5111636220705447,0.0006014425834675346,0.019517712053567154,50000.0,4.499019478878493,1400000 +0.1373693245935439,974.37,110.00242315512872,100.82018655465895,13.406757814563083,-390.77011380798024,-0.008273406773098006,55.48747313695897,1.49206576849581,2.5480276804709425,0.0005763927443686482,0.019030344859331222,50000.0,4.50154241480718,1450000 +0.13551052303970051,987.73,61.40583929888103,103.58183181577691,7.181039413874854,-393.32915254700316,0.014740533205792127,55.095901552543744,1.4834436065673857,2.5497868612290198,0.0005886525049949822,0.01915584123500009,50000.0,4.5028782473365325,1500000 +0.13714329469472034,959.41,113.55933206918753,98.48793685577337,13.179204487922247,-392.62270880126846,-0.0027471231339127405,57.9670746092986,1.5170121210062262,2.6097054821562735,0.0006813114928241742,0.02089046885696588,50000.0,4.504929471392417,1550000 +0.13022396654069404,999.48,5.173934672954423,98.24455019694945,3.2806746375204634,-397.8199331103567,-0.010066960810422994,55.71010711206481,1.4880866118287996,2.5436814350700168,0.0006038765174669909,0.019270601500877036,50000.0,4.504039791135211,1600000 +0.13439183658599899,1000.0,0.0,75.04696210658088,14.119459113612459,-395.41992509887797,0.015208881865404512,57.65585296958884,1.5047995303201867,2.6271928775179254,0.0006853462874813559,0.02092441548147117,50000.0,4.503303027176341,1650000 +0.12838196055918819,1000.0,0.0,112.35437831608917,2.656281566601329,-393.0600195739801,-0.008788978954740121,53.37916777536418,1.457315740094186,2.5120243552219774,0.0006419826494658247,0.020092470232845668,50000.0,4.5060041714890176,1700000 +0.12918855661928783,1000.0,0.0,112.4385765593994,3.1072287821880074,-395.1022312017836,-0.009324167633876269,54.40743662626275,1.474880190224645,2.537146040366915,0.0006257694888556846,0.019799267713520453,50000.0,4.504428375621994,1750000 +0.12952198005154658,983.78,67.29659426746646,110.66335485277014,8.147623969213884,-396.3370302185065,0.011351970159746745,53.96647580293696,1.4758774064445412,2.5310305599284053,0.0005318905464416834,0.018136677424528295,50000.0,4.504349735887892,1800000 +0.12918353516429593,1000.0,0.0,106.07980007636509,1.5663354051337508,-398.1317009136914,-0.018834151104800387,54.34062283893616,1.4774196389674878,2.498385812684295,0.0006057633768981237,0.019602812963030465,50000.0,4.507554428660995,1850000 +0.12519459980145117,1000.0,0.0,112.04188252207015,1.5639353308862987,-398.1962462823513,0.008960282136462832,53.88170477218653,1.4641843902849871,2.5076359104180477,0.0006302716836419493,0.02010810277239566,50000.0,4.503620319820718,1900000 +0.12693303342819245,1000.0,0.0,108.07562473525836,0.25840313802557013,-394.19435660583304,0.001304508644789549,50.751393597068315,1.443809820007095,2.4255742611956066,0.0005649457562765514,0.018916707494476367,50000.0,4.5053374140162274,1950000 +0.12165806782424407,995.77,42.087968589610036,108.54957134198415,5.088424795076612,-396.2934252355952,-0.0037823627353457084,52.55653284536341,1.4435212428665065,2.4746060013723143,0.0005588338573923741,0.01869563185354841,50000.0,4.50329709257208,2000000 +0.12770961858615287,1000.0,0.0,109.53383434417923,0.71892617745598,-393.1852081829864,-0.00868942271124566,52.97211412885641,1.4650248490536348,2.476714270944645,0.0005314329652591842,0.018293756170012603,50000.0,4.507382214144386,2050000 +0.13133313274681557,999.11,8.855388190248915,109.50317583668121,2.332444352188589,-392.3788795874043,0.019751482883915202,57.34368778514905,1.530103489464522,2.5570220642268455,0.0005231283811713334,0.018080521664317607,50000.0,4.512000071772787,2100000 +0.1296391144253309,1000.0,0.0,112.40541376660602,0.24843100873661478,-393.0675247027571,-0.012474856420867063,55.71493343616485,1.505229774481053,2.5668381137609595,0.0005749456600305681,0.0191364337543398,50000.0,4.503857562194061,2150000 +0.12592222729116656,1000.0,0.0,102.47365910030956,1.2407150209563411,-394.3508255517553,0.004712985295765047,54.961426128063,1.5006615739726907,2.5359426572560833,0.0005747277164831132,0.019011633027259212,50000.0,4.5058198872284985,2200000 +0.12021301336154351,1000.0,0.0,109.42051353572457,0.33088390095421627,-396.3669848297106,0.008454500776380233,54.961128573951,1.4824421224463076,2.558712863619301,0.0006644156658862135,0.02070153616753318,50000.0,4.507601911951305,2250000 +0.13365704155623842,1000.0,0.0,108.3082823229332,2.634475171269902,-392.9032250439509,0.002705300933718681,58.96212691555074,1.5623976728343973,2.642059254896591,0.0005647156644417205,0.018922299221586825,50000.0,4.503470878054488,2300000 +0.1252906714740383,1000.0,0.0,110.16750543203517,1.565378490926,-392.7408872973576,-0.010964081106297631,58.19192822389592,1.5103777132558966,2.627078604335743,0.0005833618965160337,0.019325315648247743,50000.0,4.499842714058784,2350000 +0.1267891293157614,1000.0,0.0,116.32237508562054,0.8850635665661963,-392.7300060656739,0.002441024071425241,55.04210586118661,1.4831958696019791,2.54097722796679,0.0006915076471745238,0.02096218256564744,50000.0,4.504277112672209,2400000 +0.1233914288432901,1000.0,0.0,112.53538425980314,1.8219857785413376,-393.66340182311944,0.003628478464968483,56.55064621429404,1.4992094802928098,2.592162832502116,0.0006994684352904178,0.02133366296591175,50000.0,4.501609133810074,2450000 +0.13230943876624135,1000.0,0.0,112.80623792130935,0.8554482235526644,-388.9339396453839,0.005781595058142975,59.82392761520365,1.5579222222864606,2.620907386703503,0.0006621258015941362,0.02057897037142221,50000.0,4.5052866650263805,2500000 +0.12719221093997476,1000.0,0.0,113.96053862689509,1.9340459675748953,-390.58447371582173,-0.0014206082331391355,59.69048191299409,1.55678125654936,2.631866437141859,0.0006881714834873329,0.020991322843666883,50000.0,4.502035439199346,2550000 +0.1246475424019989,997.4,25.869673364772108,112.19323545298137,3.7317585947085403,-391.73493743103177,-0.02230468020450321,59.49830941286102,1.5396868966615274,2.6499216994643375,0.0006510468204823496,0.020405000814041327,50000.0,4.50407030811411,2600000 +0.1282540784353022,1000.0,0.0,117.83906286416574,0.7418815469372766,-391.33301284119125,0.02113959382053496,64.95922581642154,1.6042297978425066,2.759975004301027,0.0005910669383060622,0.019340317493773993,50000.0,4.501327682900156,2650000 +0.12453447968930055,1000.0,0.0,110.38507994084745,0.539218430303223,-393.50115533020204,0.004587232915349212,64.44746730804344,1.6205505908489088,2.750969482845062,0.0005876098131978068,0.01937810448234628,50000.0,4.503649794987582,2700000 +0.12149924950286756,1000.0,0.0,118.53015824698396,1.7717861369697583,-393.30986160217697,-0.011505962618365874,62.44152712005593,1.5893549546003312,2.7229551576590714,0.0005949270964668483,0.01931736994777271,50000.0,4.502997731624529,2750000 +0.1223688934576516,1000.0,0.0,115.45422071269833,2.0080514675219563,-393.1513827777104,0.02228520902913046,63.848913867511214,1.6086615365576962,2.777106814963838,0.0006146572316284815,0.019744728512065438,50000.0,4.505432651715639,2800000 +0.12579525876924477,1000.0,0.0,119.8688817826935,1.6960552527896071,-392.57422048645225,-0.007760128022395052,67.17125047431918,1.6437253608059732,2.802713723774004,0.0006141610201247817,0.019742141531518816,50000.0,4.498944757832397,2850000 +0.11692827347695792,1000.0,0.0,121.86169346923992,1.2856831297265447,-391.9361318957502,-0.0029711144109442066,64.0294953005227,1.608069492685778,2.7707741435646724,0.0006208537180597661,0.019889948905612145,50000.0,4.50077587336936,2900000 +0.11332292782932578,1000.0,0.0,121.76957279297127,1.7793486009423063,-384.47367581542744,-0.03201123922087242,63.98691164438417,1.600277060775766,2.7489496594524354,0.0005935202216790029,0.019375214470068094,50000.0,4.501818613940051,2950000 +0.137954798468053,1000.0,0.0,120.44283051669002,1.5618122206512657,-384.4449255346699,0.06446965940035956,63.93734044914253,1.608682423269734,2.752450999426833,0.0005552736991452231,0.018658391014155767,50000.0,4.4994328445958605,3000000